The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $        */
    2 /*-
    3  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
    4  *
    5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
    6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
    7  * supported the development of this code.
    8  *
    9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
   10  *
   11  * Permission to use, copy, and modify this software with or without fee
   12  * is hereby granted, provided that this entire notice is included in
   13  * all source code copies of any software which is or includes a copy or
   14  * modification of this software.
   15  *
   16  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
   17  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
   18  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
   19  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
   20  * PURPOSE.
   21  */
   22 
   23 #include <sys/cdefs.h>
   24 __FBSDID("$FreeBSD: releng/6.0/sys/opencrypto/crypto.c 139825 2005-01-07 02:29:27Z imp $");
   25 
   26 #define CRYPTO_TIMING                           /* enable timing support */
   27 
   28 #include <sys/param.h>
   29 #include <sys/systm.h>
   30 #include <sys/eventhandler.h>
   31 #include <sys/kernel.h>
   32 #include <sys/kthread.h>
   33 #include <sys/lock.h>
   34 #include <sys/module.h>
   35 #include <sys/mutex.h>
   36 #include <sys/malloc.h>
   37 #include <sys/proc.h>
   38 #include <sys/sysctl.h>
   39 
   40 #include <vm/uma.h>
   41 #include <opencrypto/cryptodev.h>
   42 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
   43 
   44 /*
   45  * Crypto drivers register themselves by allocating a slot in the
   46  * crypto_drivers table with crypto_get_driverid() and then registering
   47  * each algorithm they support with crypto_register() and crypto_kregister().
   48  */
   49 static  struct mtx crypto_drivers_mtx;          /* lock on driver table */
   50 #define CRYPTO_DRIVER_LOCK()    mtx_lock(&crypto_drivers_mtx)
   51 #define CRYPTO_DRIVER_UNLOCK()  mtx_unlock(&crypto_drivers_mtx)
   52 static  struct cryptocap *crypto_drivers = NULL;
   53 static  int crypto_drivers_num = 0;
   54 
   55 /*
   56  * There are two queues for crypto requests; one for symmetric (e.g.
   57  * cipher) operations and one for asymmetric (e.g. MOD)operations.
   58  * A single mutex is used to lock access to both queues.  We could
   59  * have one per-queue but having one simplifies handling of block/unblock
   60  * operations.
   61  */
   62 static  TAILQ_HEAD(,cryptop) crp_q;             /* request queues */
   63 static  TAILQ_HEAD(,cryptkop) crp_kq;
   64 static  struct mtx crypto_q_mtx;
   65 #define CRYPTO_Q_LOCK()         mtx_lock(&crypto_q_mtx)
   66 #define CRYPTO_Q_UNLOCK()       mtx_unlock(&crypto_q_mtx)
   67 
   68 /*
   69  * There are two queues for processing completed crypto requests; one
   70  * for the symmetric and one for the asymmetric ops.  We only need one
   71  * but have two to avoid type futzing (cryptop vs. cryptkop).  A single
   72  * mutex is used to lock access to both queues.  Note that this lock
   73  * must be separate from the lock on request queues to insure driver
   74  * callbacks don't generate lock order reversals.
   75  */
   76 static  TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queues */
   77 static  TAILQ_HEAD(,cryptkop) crp_ret_kq;
   78 static  struct mtx crypto_ret_q_mtx;
   79 #define CRYPTO_RETQ_LOCK()      mtx_lock(&crypto_ret_q_mtx)
   80 #define CRYPTO_RETQ_UNLOCK()    mtx_unlock(&crypto_ret_q_mtx)
   81 
   82 static  uma_zone_t cryptop_zone;
   83 static  uma_zone_t cryptodesc_zone;
   84 
   85 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
   86 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
   87            &crypto_userasymcrypto, 0,
   88            "Enable/disable user-mode access to asymmetric crypto support");
   89 int     crypto_devallowsoft = 0;        /* only use hardware crypto for asym */
   90 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
   91            &crypto_devallowsoft, 0,
   92            "Enable/disable use of software asym crypto support");
   93 
   94 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
   95 
   96 static  void crypto_proc(void);
   97 static  struct proc *cryptoproc;
   98 static  void crypto_ret_proc(void);
   99 static  struct proc *cryptoretproc;
  100 static  void crypto_destroy(void);
  101 static  int crypto_invoke(struct cryptop *crp, int hint);
  102 static  int crypto_kinvoke(struct cryptkop *krp, int hint);
  103 
  104 static  struct cryptostats cryptostats;
  105 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
  106             cryptostats, "Crypto system statistics");
  107 
  108 #ifdef CRYPTO_TIMING
  109 static  int crypto_timing = 0;
  110 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
  111            &crypto_timing, 0, "Enable/disable crypto timing support");
  112 #endif
  113 
  114 static int
  115 crypto_init(void)
  116 {
  117         int error;
  118 
  119         mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
  120                 MTX_DEF|MTX_QUIET);
  121 
  122         TAILQ_INIT(&crp_q);
  123         TAILQ_INIT(&crp_kq);
  124         mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
  125 
  126         TAILQ_INIT(&crp_ret_q);
  127         TAILQ_INIT(&crp_ret_kq);
  128         mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
  129 
  130         cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
  131                                     0, 0, 0, 0,
  132                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  133         cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
  134                                     0, 0, 0, 0,
  135                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  136         if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
  137                 printf("crypto_init: cannot setup crypto zones\n");
  138                 error = ENOMEM;
  139                 goto bad;
  140         }
  141 
  142         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
  143         crypto_drivers = malloc(crypto_drivers_num *
  144             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
  145         if (crypto_drivers == NULL) {
  146                 printf("crypto_init: cannot setup crypto drivers\n");
  147                 error = ENOMEM;
  148                 goto bad;
  149         }
  150 
  151         error = kthread_create((void (*)(void *)) crypto_proc, NULL,
  152                     &cryptoproc, 0, 0, "crypto");
  153         if (error) {
  154                 printf("crypto_init: cannot start crypto thread; error %d",
  155                         error);
  156                 goto bad;
  157         }
  158 
  159         error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL,
  160                     &cryptoretproc, 0, 0, "crypto returns");
  161         if (error) {
  162                 printf("crypto_init: cannot start cryptoret thread; error %d",
  163                         error);
  164                 goto bad;
  165         }
  166         return 0;
  167 bad:
  168         crypto_destroy();
  169         return error;
  170 }
  171 
  172 /*
  173  * Signal a crypto thread to terminate.  We use the driver
  174  * table lock to synchronize the sleep/wakeups so that we
  175  * are sure the threads have terminated before we release
  176  * the data structures they use.  See crypto_finis below
  177  * for the other half of this song-and-dance.
  178  */
  179 static void
  180 crypto_terminate(struct proc **pp, void *q)
  181 {
  182         struct proc *p;
  183 
  184         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
  185         p = *pp;
  186         *pp = NULL;
  187         if (p) {
  188                 wakeup_one(q);
  189                 PROC_LOCK(p);           /* NB: insure we don't miss wakeup */
  190                 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */
  191                 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
  192                 PROC_UNLOCK(p);
  193                 CRYPTO_DRIVER_LOCK();
  194         }
  195 }
  196 
  197 static void
  198 crypto_destroy(void)
  199 {
  200         /*
  201          * Terminate any crypto threads.
  202          */
  203         CRYPTO_DRIVER_LOCK();
  204         crypto_terminate(&cryptoproc, &crp_q);
  205         crypto_terminate(&cryptoretproc, &crp_ret_q);
  206         CRYPTO_DRIVER_UNLOCK();
  207 
  208         /* XXX flush queues??? */
  209 
  210         /* 
  211          * Reclaim dynamically allocated resources.
  212          */
  213         if (crypto_drivers != NULL)
  214                 free(crypto_drivers, M_CRYPTO_DATA);
  215 
  216         if (cryptodesc_zone != NULL)
  217                 uma_zdestroy(cryptodesc_zone);
  218         if (cryptop_zone != NULL)
  219                 uma_zdestroy(cryptop_zone);
  220         mtx_destroy(&crypto_q_mtx);
  221         mtx_destroy(&crypto_ret_q_mtx);
  222         mtx_destroy(&crypto_drivers_mtx);
  223 }
  224 
  225 /*
  226  * Initialization code, both for static and dynamic loading.
  227  */
  228 static int
  229 crypto_modevent(module_t mod, int type, void *unused)
  230 {
  231         int error = EINVAL;
  232 
  233         switch (type) {
  234         case MOD_LOAD:
  235                 error = crypto_init();
  236                 if (error == 0 && bootverbose)
  237                         printf("crypto: <crypto core>\n");
  238                 break;
  239         case MOD_UNLOAD:
  240                 /*XXX disallow if active sessions */
  241                 error = 0;
  242                 crypto_destroy();
  243                 return 0;
  244         }
  245         return error;
  246 }
  247 
  248 static moduledata_t crypto_mod = {
  249         "crypto",
  250         crypto_modevent,
  251         0
  252 };
  253 MODULE_VERSION(crypto, 1);
  254 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
  255 
  256 /*
  257  * Create a new session.
  258  */
  259 int
  260 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
  261 {
  262         struct cryptoini *cr;
  263         u_int32_t hid, lid;
  264         int err = EINVAL;
  265 
  266         CRYPTO_DRIVER_LOCK();
  267 
  268         if (crypto_drivers == NULL)
  269                 goto done;
  270 
  271         /*
  272          * The algorithm we use here is pretty stupid; just use the
  273          * first driver that supports all the algorithms we need.
  274          *
  275          * XXX We need more smarts here (in real life too, but that's
  276          * XXX another story altogether).
  277          */
  278 
  279         for (hid = 0; hid < crypto_drivers_num; hid++) {
  280                 struct cryptocap *cap = &crypto_drivers[hid];
  281                 /*
  282                  * If it's not initialized or has remaining sessions
  283                  * referencing it, skip.
  284                  */
  285                 if (cap->cc_newsession == NULL ||
  286                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP))
  287                         continue;
  288 
  289                 /* Hardware required -- ignore software drivers. */
  290                 if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE))
  291                         continue;
  292                 /* Software required -- ignore hardware drivers. */
  293                 if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
  294                         continue;
  295 
  296                 /* See if all the algorithms are supported. */
  297                 for (cr = cri; cr; cr = cr->cri_next)
  298                         if (cap->cc_alg[cr->cri_alg] == 0)
  299                                 break;
  300 
  301                 if (cr == NULL) {
  302                         /* Ok, all algorithms are supported. */
  303 
  304                         /*
  305                          * Can't do everything in one session.
  306                          *
  307                          * XXX Fix this. We need to inject a "virtual" session layer right
  308                          * XXX about here.
  309                          */
  310 
  311                         /* Call the driver initialization routine. */
  312                         lid = hid;              /* Pass the driver ID. */
  313                         err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri);
  314                         if (err == 0) {
  315                                 /* XXX assert (hid &~ 0xffffff) == 0 */
  316                                 /* XXX assert (cap->cc_flags &~ 0xff) == 0 */
  317                                 (*sid) = ((cap->cc_flags & 0xff) << 24) | hid;
  318                                 (*sid) <<= 32;
  319                                 (*sid) |= (lid & 0xffffffff);
  320                                 cap->cc_sessions++;
  321                         }
  322                         break;
  323                 }
  324         }
  325 done:
  326         CRYPTO_DRIVER_UNLOCK();
  327         return err;
  328 }
  329 
  330 /*
  331  * Delete an existing session (or a reserved session on an unregistered
  332  * driver).
  333  */
  334 int
  335 crypto_freesession(u_int64_t sid)
  336 {
  337         u_int32_t hid;
  338         int err;
  339 
  340         CRYPTO_DRIVER_LOCK();
  341 
  342         if (crypto_drivers == NULL) {
  343                 err = EINVAL;
  344                 goto done;
  345         }
  346 
  347         /* Determine two IDs. */
  348         hid = CRYPTO_SESID2HID(sid);
  349 
  350         if (hid >= crypto_drivers_num) {
  351                 err = ENOENT;
  352                 goto done;
  353         }
  354 
  355         if (crypto_drivers[hid].cc_sessions)
  356                 crypto_drivers[hid].cc_sessions--;
  357 
  358         /* Call the driver cleanup routine, if available. */
  359         if (crypto_drivers[hid].cc_freesession)
  360                 err = crypto_drivers[hid].cc_freesession(
  361                                 crypto_drivers[hid].cc_arg, sid);
  362         else
  363                 err = 0;
  364 
  365         /*
  366          * If this was the last session of a driver marked as invalid,
  367          * make the entry available for reuse.
  368          */
  369         if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
  370             crypto_drivers[hid].cc_sessions == 0)
  371                 bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
  372 
  373 done:
  374         CRYPTO_DRIVER_UNLOCK();
  375         return err;
  376 }
  377 
  378 /*
  379  * Return an unused driver id.  Used by drivers prior to registering
  380  * support for the algorithms they handle.
  381  */
  382 int32_t
  383 crypto_get_driverid(u_int32_t flags)
  384 {
  385         struct cryptocap *newdrv;
  386         int i;
  387 
  388         CRYPTO_DRIVER_LOCK();
  389 
  390         for (i = 0; i < crypto_drivers_num; i++)
  391                 if (crypto_drivers[i].cc_process == NULL &&
  392                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
  393                     crypto_drivers[i].cc_sessions == 0)
  394                         break;
  395 
  396         /* Out of entries, allocate some more. */
  397         if (i == crypto_drivers_num) {
  398                 /* Be careful about wrap-around. */
  399                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
  400                         CRYPTO_DRIVER_UNLOCK();
  401                         printf("crypto: driver count wraparound!\n");
  402                         return -1;
  403                 }
  404 
  405                 newdrv = malloc(2 * crypto_drivers_num *
  406                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
  407                 if (newdrv == NULL) {
  408                         CRYPTO_DRIVER_UNLOCK();
  409                         printf("crypto: no space to expand driver table!\n");
  410                         return -1;
  411                 }
  412 
  413                 bcopy(crypto_drivers, newdrv,
  414                     crypto_drivers_num * sizeof(struct cryptocap));
  415 
  416                 crypto_drivers_num *= 2;
  417 
  418                 free(crypto_drivers, M_CRYPTO_DATA);
  419                 crypto_drivers = newdrv;
  420         }
  421 
  422         /* NB: state is zero'd on free */
  423         crypto_drivers[i].cc_sessions = 1;      /* Mark */
  424         crypto_drivers[i].cc_flags = flags;
  425         if (bootverbose)
  426                 printf("crypto: assign driver %u, flags %u\n", i, flags);
  427 
  428         CRYPTO_DRIVER_UNLOCK();
  429 
  430         return i;
  431 }
  432 
  433 static struct cryptocap *
  434 crypto_checkdriver(u_int32_t hid)
  435 {
  436         if (crypto_drivers == NULL)
  437                 return NULL;
  438         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
  439 }
  440 
  441 /*
  442  * Register support for a key-related algorithm.  This routine
  443  * is called once for each algorithm supported a driver.
  444  */
  445 int
  446 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
  447     int (*kprocess)(void*, struct cryptkop *, int),
  448     void *karg)
  449 {
  450         struct cryptocap *cap;
  451         int err;
  452 
  453         CRYPTO_DRIVER_LOCK();
  454 
  455         cap = crypto_checkdriver(driverid);
  456         if (cap != NULL &&
  457             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
  458                 /*
  459                  * XXX Do some performance testing to determine placing.
  460                  * XXX We probably need an auxiliary data structure that
  461                  * XXX describes relative performances.
  462                  */
  463 
  464                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
  465                 if (bootverbose)
  466                         printf("crypto: driver %u registers key alg %u flags %u\n"
  467                                 , driverid
  468                                 , kalg
  469                                 , flags
  470                         );
  471 
  472                 if (cap->cc_kprocess == NULL) {
  473                         cap->cc_karg = karg;
  474                         cap->cc_kprocess = kprocess;
  475                 }
  476                 err = 0;
  477         } else
  478                 err = EINVAL;
  479 
  480         CRYPTO_DRIVER_UNLOCK();
  481         return err;
  482 }
  483 
  484 /*
  485  * Register support for a non-key-related algorithm.  This routine
  486  * is called once for each such algorithm supported by a driver.
  487  */
  488 int
  489 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
  490     u_int32_t flags,
  491     int (*newses)(void*, u_int32_t*, struct cryptoini*),
  492     int (*freeses)(void*, u_int64_t),
  493     int (*process)(void*, struct cryptop *, int),
  494     void *arg)
  495 {
  496         struct cryptocap *cap;
  497         int err;
  498 
  499         CRYPTO_DRIVER_LOCK();
  500 
  501         cap = crypto_checkdriver(driverid);
  502         /* NB: algorithms are in the range [1..max] */
  503         if (cap != NULL &&
  504             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
  505                 /*
  506                  * XXX Do some performance testing to determine placing.
  507                  * XXX We probably need an auxiliary data structure that
  508                  * XXX describes relative performances.
  509                  */
  510 
  511                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
  512                 cap->cc_max_op_len[alg] = maxoplen;
  513                 if (bootverbose)
  514                         printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
  515                                 , driverid
  516                                 , alg
  517                                 , flags
  518                                 , maxoplen
  519                         );
  520 
  521                 if (cap->cc_process == NULL) {
  522                         cap->cc_arg = arg;
  523                         cap->cc_newsession = newses;
  524                         cap->cc_process = process;
  525                         cap->cc_freesession = freeses;
  526                         cap->cc_sessions = 0;           /* Unmark */
  527                 }
  528                 err = 0;
  529         } else
  530                 err = EINVAL;
  531 
  532         CRYPTO_DRIVER_UNLOCK();
  533         return err;
  534 }
  535 
  536 /*
  537  * Unregister a crypto driver. If there are pending sessions using it,
  538  * leave enough information around so that subsequent calls using those
  539  * sessions will correctly detect the driver has been unregistered and
  540  * reroute requests.
  541  */
  542 int
  543 crypto_unregister(u_int32_t driverid, int alg)
  544 {
  545         int i, err;
  546         u_int32_t ses;
  547         struct cryptocap *cap;
  548 
  549         CRYPTO_DRIVER_LOCK();
  550 
  551         cap = crypto_checkdriver(driverid);
  552         if (cap != NULL &&
  553             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
  554             cap->cc_alg[alg] != 0) {
  555                 cap->cc_alg[alg] = 0;
  556                 cap->cc_max_op_len[alg] = 0;
  557 
  558                 /* Was this the last algorithm ? */
  559                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
  560                         if (cap->cc_alg[i] != 0)
  561                                 break;
  562 
  563                 if (i == CRYPTO_ALGORITHM_MAX + 1) {
  564                         ses = cap->cc_sessions;
  565                         bzero(cap, sizeof(struct cryptocap));
  566                         if (ses != 0) {
  567                                 /*
  568                                  * If there are pending sessions, just mark as invalid.
  569                                  */
  570                                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
  571                                 cap->cc_sessions = ses;
  572                         }
  573                 }
  574                 err = 0;
  575         } else
  576                 err = EINVAL;
  577 
  578         CRYPTO_DRIVER_UNLOCK();
  579         return err;
  580 }
  581 
  582 /*
  583  * Unregister all algorithms associated with a crypto driver.
  584  * If there are pending sessions using it, leave enough information
  585  * around so that subsequent calls using those sessions will
  586  * correctly detect the driver has been unregistered and reroute
  587  * requests.
  588  */
  589 int
  590 crypto_unregister_all(u_int32_t driverid)
  591 {
  592         int i, err;
  593         u_int32_t ses;
  594         struct cryptocap *cap;
  595 
  596         CRYPTO_DRIVER_LOCK();
  597 
  598         cap = crypto_checkdriver(driverid);
  599         if (cap != NULL) {
  600                 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
  601                         cap->cc_alg[i] = 0;
  602                         cap->cc_max_op_len[i] = 0;
  603                 }
  604                 ses = cap->cc_sessions;
  605                 bzero(cap, sizeof(struct cryptocap));
  606                 if (ses != 0) {
  607                         /*
  608                          * If there are pending sessions, just mark as invalid.
  609                          */
  610                         cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
  611                         cap->cc_sessions = ses;
  612                 }
  613                 err = 0;
  614         } else
  615                 err = EINVAL;
  616 
  617         CRYPTO_DRIVER_UNLOCK();
  618         return err;
  619 }
  620 
  621 /*
  622  * Clear blockage on a driver.  The what parameter indicates whether
  623  * the driver is now ready for cryptop's and/or cryptokop's.
  624  */
  625 int
  626 crypto_unblock(u_int32_t driverid, int what)
  627 {
  628         struct cryptocap *cap;
  629         int needwakeup, err;
  630 
  631         CRYPTO_Q_LOCK();
  632         cap = crypto_checkdriver(driverid);
  633         if (cap != NULL) {
  634                 needwakeup = 0;
  635                 if (what & CRYPTO_SYMQ) {
  636                         needwakeup |= cap->cc_qblocked;
  637                         cap->cc_qblocked = 0;
  638                 }
  639                 if (what & CRYPTO_ASYMQ) {
  640                         needwakeup |= cap->cc_kqblocked;
  641                         cap->cc_kqblocked = 0;
  642                 }
  643                 if (needwakeup)
  644                         wakeup_one(&crp_q);
  645                 err = 0;
  646         } else
  647                 err = EINVAL;
  648         CRYPTO_Q_UNLOCK();
  649 
  650         return err;
  651 }
  652 
  653 /*
  654  * Add a crypto request to a queue, to be processed by the kernel thread.
  655  */
  656 int
  657 crypto_dispatch(struct cryptop *crp)
  658 {
  659         u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid);
  660         int result;
  661 
  662         cryptostats.cs_ops++;
  663 
  664 #ifdef CRYPTO_TIMING
  665         if (crypto_timing)
  666                 binuptime(&crp->crp_tstamp);
  667 #endif
  668 
  669         CRYPTO_Q_LOCK();
  670         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
  671                 struct cryptocap *cap;
  672                 /*
  673                  * Caller marked the request to be processed
  674                  * immediately; dispatch it directly to the
  675                  * driver unless the driver is currently blocked.
  676                  */
  677                 cap = crypto_checkdriver(hid);
  678                 if (cap && !cap->cc_qblocked) {
  679                         result = crypto_invoke(crp, 0);
  680                         if (result == ERESTART) {
  681                                 /*
  682                                  * The driver ran out of resources, mark the
  683                                  * driver ``blocked'' for cryptop's and put
  684                                  * the request on the queue.
  685                                  *
  686                                  * XXX ops are placed at the tail so their
  687                                  * order is preserved but this can place them
  688                                  * behind batch'd ops.
  689                                  */
  690                                 crypto_drivers[hid].cc_qblocked = 1;
  691                                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
  692                                 cryptostats.cs_blocks++;
  693                                 result = 0;
  694                         }
  695                 } else {
  696                         /*
  697                          * The driver is blocked, just queue the op until
  698                          * it unblocks and the kernel thread gets kicked.
  699                          */
  700                         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
  701                         result = 0;
  702                 }
  703         } else {
  704                 int wasempty;
  705                 /*
  706                  * Caller marked the request as ``ok to delay'';
  707                  * queue it for the dispatch thread.  This is desirable
  708                  * when the operation is low priority and/or suitable
  709                  * for batching.
  710                  */
  711                 wasempty = TAILQ_EMPTY(&crp_q);
  712                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
  713                 if (wasempty)
  714                         wakeup_one(&crp_q);
  715                 result = 0;
  716         }
  717         CRYPTO_Q_UNLOCK();
  718 
  719         return result;
  720 }
  721 
  722 /*
  723  * Add an asymetric crypto request to a queue,
  724  * to be processed by the kernel thread.
  725  */
  726 int
  727 crypto_kdispatch(struct cryptkop *krp)
  728 {
  729         struct cryptocap *cap;
  730         int result;
  731 
  732         cryptostats.cs_kops++;
  733 
  734         CRYPTO_Q_LOCK();
  735         cap = crypto_checkdriver(krp->krp_hid);
  736         if (cap && !cap->cc_kqblocked) {
  737                 result = crypto_kinvoke(krp, 0);
  738                 if (result == ERESTART) {
  739                         /*
  740                          * The driver ran out of resources, mark the
  741                          * driver ``blocked'' for cryptkop's and put
  742                          * the request back in the queue.  It would
  743                          * best to put the request back where we got
  744                          * it but that's hard so for now we put it
  745                          * at the front.  This should be ok; putting
  746                          * it at the end does not work.
  747                          */
  748                         crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
  749                         TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
  750                         cryptostats.cs_kblocks++;
  751                 }
  752         } else {
  753                 /*
  754                  * The driver is blocked, just queue the op until
  755                  * it unblocks and the kernel thread gets kicked.
  756                  */
  757                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
  758                 result = 0;
  759         }
  760         CRYPTO_Q_UNLOCK();
  761 
  762         return result;
  763 }
  764 
  765 /*
  766  * Dispatch an assymetric crypto request to the appropriate crypto devices.
  767  */
  768 static int
  769 crypto_kinvoke(struct cryptkop *krp, int hint)
  770 {
  771         u_int32_t hid;
  772         int error;
  773 
  774         mtx_assert(&crypto_q_mtx, MA_OWNED);
  775 
  776         /* Sanity checks. */
  777         if (krp == NULL)
  778                 return EINVAL;
  779         if (krp->krp_callback == NULL) {
  780                 free(krp, M_XDATA);             /* XXX allocated in cryptodev */
  781                 return EINVAL;
  782         }
  783 
  784         for (hid = 0; hid < crypto_drivers_num; hid++) {
  785                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
  786                     !crypto_devallowsoft)
  787                         continue;
  788                 if (crypto_drivers[hid].cc_kprocess == NULL)
  789                         continue;
  790                 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
  791                     CRYPTO_ALG_FLAG_SUPPORTED) == 0)
  792                         continue;
  793                 break;
  794         }
  795         if (hid < crypto_drivers_num) {
  796                 krp->krp_hid = hid;
  797                 error = crypto_drivers[hid].cc_kprocess(
  798                                 crypto_drivers[hid].cc_karg, krp, hint);
  799         } else
  800                 error = ENODEV;
  801 
  802         if (error) {
  803                 krp->krp_status = error;
  804                 crypto_kdone(krp);
  805         }
  806         return 0;
  807 }
  808 
  809 #ifdef CRYPTO_TIMING
  810 static void
  811 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
  812 {
  813         struct bintime now, delta;
  814         struct timespec t;
  815         uint64_t u;
  816 
  817         binuptime(&now);
  818         u = now.frac;
  819         delta.frac = now.frac - bt->frac;
  820         delta.sec = now.sec - bt->sec;
  821         if (u < delta.frac)
  822                 delta.sec--;
  823         bintime2timespec(&delta, &t);
  824         timespecadd(&ts->acc, &t);
  825         if (timespeccmp(&t, &ts->min, <))
  826                 ts->min = t;
  827         if (timespeccmp(&t, &ts->max, >))
  828                 ts->max = t;
  829         ts->count++;
  830 
  831         *bt = now;
  832 }
  833 #endif
  834 
  835 /*
  836  * Dispatch a crypto request to the appropriate crypto devices.
  837  */
  838 static int
  839 crypto_invoke(struct cryptop *crp, int hint)
  840 {
  841         u_int32_t hid;
  842         int (*process)(void*, struct cryptop *, int);
  843 
  844 #ifdef CRYPTO_TIMING
  845         if (crypto_timing)
  846                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
  847 #endif
  848         /* Sanity checks. */
  849         if (crp == NULL)
  850                 return EINVAL;
  851         if (crp->crp_callback == NULL) {
  852                 crypto_freereq(crp);
  853                 return EINVAL;
  854         }
  855         if (crp->crp_desc == NULL) {
  856                 crp->crp_etype = EINVAL;
  857                 crypto_done(crp);
  858                 return 0;
  859         }
  860 
  861         hid = CRYPTO_SESID2HID(crp->crp_sid);
  862         if (hid < crypto_drivers_num) {
  863                 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
  864                         crypto_freesession(crp->crp_sid);
  865                 process = crypto_drivers[hid].cc_process;
  866         } else {
  867                 process = NULL;
  868         }
  869 
  870         if (process == NULL) {
  871                 struct cryptodesc *crd;
  872                 u_int64_t nid;
  873 
  874                 /*
  875                  * Driver has unregistered; migrate the session and return
  876                  * an error to the caller so they'll resubmit the op.
  877                  */
  878                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
  879                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
  880 
  881                 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
  882                         crp->crp_sid = nid;
  883 
  884                 crp->crp_etype = EAGAIN;
  885                 crypto_done(crp);
  886                 return 0;
  887         } else {
  888                 /*
  889                  * Invoke the driver to process the request.
  890                  */
  891                 return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
  892         }
  893 }
  894 
  895 /*
  896  * Release a set of crypto descriptors.
  897  */
  898 void
  899 crypto_freereq(struct cryptop *crp)
  900 {
  901         struct cryptodesc *crd;
  902 
  903         if (crp == NULL)
  904                 return;
  905 
  906         while ((crd = crp->crp_desc) != NULL) {
  907                 crp->crp_desc = crd->crd_next;
  908                 uma_zfree(cryptodesc_zone, crd);
  909         }
  910 
  911         uma_zfree(cryptop_zone, crp);
  912 }
  913 
  914 /*
  915  * Acquire a set of crypto descriptors.
  916  */
  917 struct cryptop *
  918 crypto_getreq(int num)
  919 {
  920         struct cryptodesc *crd;
  921         struct cryptop *crp;
  922 
  923         crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
  924         if (crp != NULL) {
  925                 while (num--) {
  926                         crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
  927                         if (crd == NULL) {
  928                                 crypto_freereq(crp);
  929                                 return NULL;
  930                         }
  931 
  932                         crd->crd_next = crp->crp_desc;
  933                         crp->crp_desc = crd;
  934                 }
  935         }
  936         return crp;
  937 }
  938 
  939 /*
  940  * Invoke the callback on behalf of the driver.
  941  */
  942 void
  943 crypto_done(struct cryptop *crp)
  944 {
  945         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
  946                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
  947         crp->crp_flags |= CRYPTO_F_DONE;
  948         if (crp->crp_etype != 0)
  949                 cryptostats.cs_errs++;
  950 #ifdef CRYPTO_TIMING
  951         if (crypto_timing)
  952                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
  953 #endif
  954         /*
  955          * CBIMM means unconditionally do the callback immediately;
  956          * CBIFSYNC means do the callback immediately only if the
  957          * operation was done synchronously.  Both are used to avoid
  958          * doing extraneous context switches; the latter is mostly
  959          * used with the software crypto driver.
  960          */
  961         if ((crp->crp_flags & CRYPTO_F_CBIMM) ||
  962             ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
  963              (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) {
  964                 /*
  965                  * Do the callback directly.  This is ok when the
  966                  * callback routine does very little (e.g. the
  967                  * /dev/crypto callback method just does a wakeup).
  968                  */
  969 #ifdef CRYPTO_TIMING
  970                 if (crypto_timing) {
  971                         /*
  972                          * NB: We must copy the timestamp before
  973                          * doing the callback as the cryptop is
  974                          * likely to be reclaimed.
  975                          */
  976                         struct bintime t = crp->crp_tstamp;
  977                         crypto_tstat(&cryptostats.cs_cb, &t);
  978                         crp->crp_callback(crp);
  979                         crypto_tstat(&cryptostats.cs_finis, &t);
  980                 } else
  981 #endif
  982                         crp->crp_callback(crp);
  983         } else {
  984                 int wasempty;
  985                 /*
  986                  * Normal case; queue the callback for the thread.
  987                  */
  988                 CRYPTO_RETQ_LOCK();
  989                 wasempty = TAILQ_EMPTY(&crp_ret_q);
  990                 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
  991 
  992                 if (wasempty)
  993                         wakeup_one(&crp_ret_q); /* shared wait channel */
  994                 CRYPTO_RETQ_UNLOCK();
  995         }
  996 }
  997 
  998 /*
  999  * Invoke the callback on behalf of the driver.
 1000  */
 1001 void
 1002 crypto_kdone(struct cryptkop *krp)
 1003 {
 1004         int wasempty;
 1005 
 1006         if (krp->krp_status != 0)
 1007                 cryptostats.cs_kerrs++;
 1008         CRYPTO_RETQ_LOCK();
 1009         wasempty = TAILQ_EMPTY(&crp_ret_kq);
 1010         TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
 1011 
 1012         if (wasempty)
 1013                 wakeup_one(&crp_ret_q);         /* shared wait channel */
 1014         CRYPTO_RETQ_UNLOCK();
 1015 }
 1016 
 1017 int
 1018 crypto_getfeat(int *featp)
 1019 {
 1020         int hid, kalg, feat = 0;
 1021 
 1022         if (!crypto_userasymcrypto)
 1023                 goto out;         
 1024 
 1025         CRYPTO_DRIVER_LOCK();
 1026         for (hid = 0; hid < crypto_drivers_num; hid++) {
 1027                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
 1028                     !crypto_devallowsoft) {
 1029                         continue;
 1030                 }
 1031                 if (crypto_drivers[hid].cc_kprocess == NULL)
 1032                         continue;
 1033                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
 1034                         if ((crypto_drivers[hid].cc_kalg[kalg] &
 1035                             CRYPTO_ALG_FLAG_SUPPORTED) != 0)
 1036                                 feat |=  1 << kalg;
 1037         }
 1038         CRYPTO_DRIVER_UNLOCK();
 1039 out:
 1040         *featp = feat;
 1041         return (0);
 1042 }
 1043 
 1044 /*
 1045  * Terminate a thread at module unload.  The process that
 1046  * initiated this is waiting for us to signal that we're gone;
 1047  * wake it up and exit.  We use the driver table lock to insure
 1048  * we don't do the wakeup before they're waiting.  There is no
 1049  * race here because the waiter sleeps on the proc lock for the
 1050  * thread so it gets notified at the right time because of an
 1051  * extra wakeup that's done in exit1().
 1052  */
 1053 static void
 1054 crypto_finis(void *chan)
 1055 {
 1056         CRYPTO_DRIVER_LOCK();
 1057         wakeup_one(chan);
 1058         CRYPTO_DRIVER_UNLOCK();
 1059         kthread_exit(0);
 1060 }
 1061 
 1062 /*
 1063  * Crypto thread, dispatches crypto requests.
 1064  */
 1065 static void
 1066 crypto_proc(void)
 1067 {
 1068         struct cryptop *crp, *submit;
 1069         struct cryptkop *krp;
 1070         struct cryptocap *cap;
 1071         int result, hint;
 1072 
 1073         CRYPTO_Q_LOCK();
 1074         for (;;) {
 1075                 /*
 1076                  * Find the first element in the queue that can be
 1077                  * processed and look-ahead to see if multiple ops
 1078                  * are ready for the same driver.
 1079                  */
 1080                 submit = NULL;
 1081                 hint = 0;
 1082                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
 1083                         u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid);
 1084                         cap = crypto_checkdriver(hid);
 1085                         if (cap == NULL || cap->cc_process == NULL) {
 1086                                 /* Op needs to be migrated, process it. */
 1087                                 if (submit == NULL)
 1088                                         submit = crp;
 1089                                 break;
 1090                         }
 1091                         if (!cap->cc_qblocked) {
 1092                                 if (submit != NULL) {
 1093                                         /*
 1094                                          * We stop on finding another op,
 1095                                          * regardless whether its for the same
 1096                                          * driver or not.  We could keep
 1097                                          * searching the queue but it might be
 1098                                          * better to just use a per-driver
 1099                                          * queue instead.
 1100                                          */
 1101                                         if (CRYPTO_SESID2HID(submit->crp_sid) == hid)
 1102                                                 hint = CRYPTO_HINT_MORE;
 1103                                         break;
 1104                                 } else {
 1105                                         submit = crp;
 1106                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
 1107                                                 break;
 1108                                         /* keep scanning for more are q'd */
 1109                                 }
 1110                         }
 1111                 }
 1112                 if (submit != NULL) {
 1113                         TAILQ_REMOVE(&crp_q, submit, crp_next);
 1114                         result = crypto_invoke(submit, hint);
 1115                         if (result == ERESTART) {
 1116                                 /*
 1117                                  * The driver ran out of resources, mark the
 1118                                  * driver ``blocked'' for cryptop's and put
 1119                                  * the request back in the queue.  It would
 1120                                  * best to put the request back where we got
 1121                                  * it but that's hard so for now we put it
 1122                                  * at the front.  This should be ok; putting
 1123                                  * it at the end does not work.
 1124                                  */
 1125                                 /* XXX validate sid again? */
 1126                                 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
 1127                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
 1128                                 cryptostats.cs_blocks++;
 1129                         }
 1130                 }
 1131 
 1132                 /* As above, but for key ops */
 1133                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
 1134                         cap = crypto_checkdriver(krp->krp_hid);
 1135                         if (cap == NULL || cap->cc_kprocess == NULL) {
 1136                                 /* Op needs to be migrated, process it. */
 1137                                 break;
 1138                         }
 1139                         if (!cap->cc_kqblocked)
 1140                                 break;
 1141                 }
 1142                 if (krp != NULL) {
 1143                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
 1144                         result = crypto_kinvoke(krp, 0);
 1145                         if (result == ERESTART) {
 1146                                 /*
 1147                                  * The driver ran out of resources, mark the
 1148                                  * driver ``blocked'' for cryptkop's and put
 1149                                  * the request back in the queue.  It would
 1150                                  * best to put the request back where we got
 1151                                  * it but that's hard so for now we put it
 1152                                  * at the front.  This should be ok; putting
 1153                                  * it at the end does not work.
 1154                                  */
 1155                                 /* XXX validate sid again? */
 1156                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
 1157                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
 1158                                 cryptostats.cs_kblocks++;
 1159                         }
 1160                 }
 1161 
 1162                 if (submit == NULL && krp == NULL) {
 1163                         /*
 1164                          * Nothing more to be processed.  Sleep until we're
 1165                          * woken because there are more ops to process.
 1166                          * This happens either by submission or by a driver
 1167                          * becoming unblocked and notifying us through
 1168                          * crypto_unblock.  Note that when we wakeup we
 1169                          * start processing each queue again from the
 1170                          * front. It's not clear that it's important to
 1171                          * preserve this ordering since ops may finish
 1172                          * out of order if dispatched to different devices
 1173                          * and some become blocked while others do not.
 1174                          */
 1175                         msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
 1176                         if (cryptoproc == NULL)
 1177                                 break;
 1178                         cryptostats.cs_intrs++;
 1179                 }
 1180         }
 1181         CRYPTO_Q_UNLOCK();
 1182 
 1183         crypto_finis(&crp_q);
 1184 }
 1185 
 1186 /*
 1187  * Crypto returns thread, does callbacks for processed crypto requests.
 1188  * Callbacks are done here, rather than in the crypto drivers, because
 1189  * callbacks typically are expensive and would slow interrupt handling.
 1190  */
 1191 static void
 1192 crypto_ret_proc(void)
 1193 {
 1194         struct cryptop *crpt;
 1195         struct cryptkop *krpt;
 1196 
 1197         CRYPTO_RETQ_LOCK();
 1198         for (;;) {
 1199                 /* Harvest return q's for completed ops */
 1200                 crpt = TAILQ_FIRST(&crp_ret_q);
 1201                 if (crpt != NULL)
 1202                         TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
 1203 
 1204                 krpt = TAILQ_FIRST(&crp_ret_kq);
 1205                 if (krpt != NULL)
 1206                         TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
 1207 
 1208                 if (crpt != NULL || krpt != NULL) {
 1209                         CRYPTO_RETQ_UNLOCK();
 1210                         /*
 1211                          * Run callbacks unlocked.
 1212                          */
 1213                         if (crpt != NULL) {
 1214 #ifdef CRYPTO_TIMING
 1215                                 if (crypto_timing) {
 1216                                         /*
 1217                                          * NB: We must copy the timestamp before
 1218                                          * doing the callback as the cryptop is
 1219                                          * likely to be reclaimed.
 1220                                          */
 1221                                         struct bintime t = crpt->crp_tstamp;
 1222                                         crypto_tstat(&cryptostats.cs_cb, &t);
 1223                                         crpt->crp_callback(crpt);
 1224                                         crypto_tstat(&cryptostats.cs_finis, &t);
 1225                                 } else
 1226 #endif
 1227                                         crpt->crp_callback(crpt);
 1228                         }
 1229                         if (krpt != NULL)
 1230                                 krpt->krp_callback(krpt);
 1231                         CRYPTO_RETQ_LOCK();
 1232                 } else {
 1233                         /*
 1234                          * Nothing more to be processed.  Sleep until we're
 1235                          * woken because there are more returns to process.
 1236                          */
 1237                         msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
 1238                                 "crypto_ret_wait", 0);
 1239                         if (cryptoretproc == NULL)
 1240                                 break;
 1241                         cryptostats.cs_rets++;
 1242                 }
 1243         }
 1244         CRYPTO_RETQ_UNLOCK();
 1245 
 1246         crypto_finis(&crp_ret_q);
 1247 }

Cache object: 72c3ac84fc3fd1af0739f7838b3a604e


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.