The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  *
   13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   23  */
   24 
   25 #include <sys/cdefs.h>
   26 __FBSDID("$FreeBSD: releng/12.0/sys/opencrypto/crypto.c 336914 2018-07-30 15:46:40Z asomers $");
   27 
   28 /*
   29  * Cryptographic Subsystem.
   30  *
   31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
   32  * that has the copyright shown below.  Very little of the original
   33  * code remains.
   34  */
   35 
   36 /*-
   37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   38  *
   39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   41  * supported the development of this code.
   42  *
   43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
   44  *
   45  * Permission to use, copy, and modify this software with or without fee
   46  * is hereby granted, provided that this entire notice is included in
   47  * all source code copies of any software which is or includes a copy or
   48  * modification of this software.
   49  *
   50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
   51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
   52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
   53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
   54  * PURPOSE.
   55  */
   56 
   57 #define CRYPTO_TIMING                           /* enable timing support */
   58 
   59 #include "opt_ddb.h"
   60 
   61 #include <sys/param.h>
   62 #include <sys/systm.h>
   63 #include <sys/eventhandler.h>
   64 #include <sys/kernel.h>
   65 #include <sys/kthread.h>
   66 #include <sys/linker.h>
   67 #include <sys/lock.h>
   68 #include <sys/module.h>
   69 #include <sys/mutex.h>
   70 #include <sys/malloc.h>
   71 #include <sys/proc.h>
   72 #include <sys/sdt.h>
   73 #include <sys/smp.h>
   74 #include <sys/sysctl.h>
   75 #include <sys/taskqueue.h>
   76 
   77 #include <ddb/ddb.h>
   78 
   79 #include <vm/uma.h>
   80 #include <crypto/intake.h>
   81 #include <opencrypto/cryptodev.h>
   82 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
   83 
   84 #include <sys/kobj.h>
   85 #include <sys/bus.h>
   86 #include "cryptodev_if.h"
   87 
   88 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
   89 #include <machine/pcb.h>
   90 #endif
   91 
   92 struct crypto_session {
   93         device_t parent;
   94         void *softc;
   95         uint32_t hid;
   96         uint32_t capabilities;
   97 };
   98 
   99 SDT_PROVIDER_DEFINE(opencrypto);
  100 
  101 /*
  102  * Crypto drivers register themselves by allocating a slot in the
  103  * crypto_drivers table with crypto_get_driverid() and then registering
  104  * each algorithm they support with crypto_register() and crypto_kregister().
  105  */
  106 static  struct mtx crypto_drivers_mtx;          /* lock on driver table */
  107 #define CRYPTO_DRIVER_LOCK()    mtx_lock(&crypto_drivers_mtx)
  108 #define CRYPTO_DRIVER_UNLOCK()  mtx_unlock(&crypto_drivers_mtx)
  109 #define CRYPTO_DRIVER_ASSERT()  mtx_assert(&crypto_drivers_mtx, MA_OWNED)
  110 
  111 /*
  112  * Crypto device/driver capabilities structure.
  113  *
  114  * Synchronization:
  115  * (d) - protected by CRYPTO_DRIVER_LOCK()
  116  * (q) - protected by CRYPTO_Q_LOCK()
  117  * Not tagged fields are read-only.
  118  */
  119 struct cryptocap {
  120         device_t        cc_dev;                 /* (d) device/driver */
  121         u_int32_t       cc_sessions;            /* (d) # of sessions */
  122         u_int32_t       cc_koperations;         /* (d) # os asym operations */
  123         /*
  124          * Largest possible operator length (in bits) for each type of
  125          * encryption algorithm. XXX not used
  126          */
  127         u_int16_t       cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
  128         u_int8_t        cc_alg[CRYPTO_ALGORITHM_MAX + 1];
  129         u_int8_t        cc_kalg[CRK_ALGORITHM_MAX + 1];
  130 
  131         int             cc_flags;               /* (d) flags */
  132 #define CRYPTOCAP_F_CLEANUP     0x80000000      /* needs resource cleanup */
  133         int             cc_qblocked;            /* (q) symmetric q blocked */
  134         int             cc_kqblocked;           /* (q) asymmetric q blocked */
  135         size_t          cc_session_size;
  136 };
  137 static  struct cryptocap *crypto_drivers = NULL;
  138 static  int crypto_drivers_num = 0;
  139 
  140 /*
  141  * There are two queues for crypto requests; one for symmetric (e.g.
  142  * cipher) operations and one for asymmetric (e.g. MOD)operations.
  143  * A single mutex is used to lock access to both queues.  We could
  144  * have one per-queue but having one simplifies handling of block/unblock
  145  * operations.
  146  */
  147 static  int crp_sleep = 0;
  148 static  TAILQ_HEAD(cryptop_q ,cryptop) crp_q;           /* request queues */
  149 static  TAILQ_HEAD(,cryptkop) crp_kq;
  150 static  struct mtx crypto_q_mtx;
  151 #define CRYPTO_Q_LOCK()         mtx_lock(&crypto_q_mtx)
  152 #define CRYPTO_Q_UNLOCK()       mtx_unlock(&crypto_q_mtx)
  153 
  154 /*
  155  * Taskqueue used to dispatch the crypto requests
  156  * that have the CRYPTO_F_ASYNC flag
  157  */
  158 static struct taskqueue *crypto_tq;
  159 
  160 /*
  161  * Crypto seq numbers are operated on with modular arithmetic
  162  */
  163 #define CRYPTO_SEQ_GT(a,b)      ((int)((a)-(b)) > 0)
  164 
  165 struct crypto_ret_worker {
  166         struct mtx crypto_ret_mtx;
  167 
  168         TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
  169         TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queue for symetric jobs */
  170         TAILQ_HEAD(,cryptkop) crp_ret_kq;       /* callback queue for asym jobs */
  171 
  172         u_int32_t reorder_ops;          /* total ordered sym jobs received */
  173         u_int32_t reorder_cur_seq;      /* current sym job dispatched */
  174 
  175         struct proc *cryptoretproc;
  176 };
  177 static struct crypto_ret_worker *crypto_ret_workers = NULL;
  178 
  179 #define CRYPTO_RETW(i)          (&crypto_ret_workers[i])
  180 #define CRYPTO_RETW_ID(w)       ((w) - crypto_ret_workers)
  181 #define FOREACH_CRYPTO_RETW(w) \
  182         for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
  183 
  184 #define CRYPTO_RETW_LOCK(w)     mtx_lock(&w->crypto_ret_mtx)
  185 #define CRYPTO_RETW_UNLOCK(w)   mtx_unlock(&w->crypto_ret_mtx)
  186 #define CRYPTO_RETW_EMPTY(w) \
  187         (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
  188 
  189 static int crypto_workers_num = 0;
  190 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
  191            &crypto_workers_num, 0,
  192            "Number of crypto workers used to dispatch crypto jobs");
  193 
  194 static  uma_zone_t cryptop_zone;
  195 static  uma_zone_t cryptodesc_zone;
  196 static  uma_zone_t cryptoses_zone;
  197 
  198 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
  199 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
  200            &crypto_userasymcrypto, 0,
  201            "Enable/disable user-mode access to asymmetric crypto support");
  202 int     crypto_devallowsoft = 0;        /* only use hardware crypto */
  203 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
  204            &crypto_devallowsoft, 0,
  205            "Enable/disable use of software crypto by /dev/crypto");
  206 
  207 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
  208 
  209 static  void crypto_proc(void);
  210 static  struct proc *cryptoproc;
  211 static  void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
  212 static  void crypto_destroy(void);
  213 static  int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
  214 static  int crypto_kinvoke(struct cryptkop *krp, int flags);
  215 static  void crypto_remove(struct cryptocap *cap);
  216 static  void crypto_task_invoke(void *ctx, int pending);
  217 static void crypto_batch_enqueue(struct cryptop *crp);
  218 
  219 static  struct cryptostats cryptostats;
  220 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
  221             cryptostats, "Crypto system statistics");
  222 
  223 #ifdef CRYPTO_TIMING
  224 static  int crypto_timing = 0;
  225 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
  226            &crypto_timing, 0, "Enable/disable crypto timing support");
  227 #endif
  228 
  229 /* Try to avoid directly exposing the key buffer as a symbol */
  230 static struct keybuf *keybuf;
  231 
  232 static struct keybuf empty_keybuf = {
  233         .kb_nents = 0
  234 };
  235 
  236 /* Obtain the key buffer from boot metadata */
  237 static void
  238 keybuf_init(void)
  239 {
  240         caddr_t kmdp;
  241 
  242         kmdp = preload_search_by_type("elf kernel");
  243 
  244         if (kmdp == NULL)
  245                 kmdp = preload_search_by_type("elf64 kernel");
  246 
  247         keybuf = (struct keybuf *)preload_search_info(kmdp,
  248             MODINFO_METADATA | MODINFOMD_KEYBUF);
  249 
  250         if (keybuf == NULL)
  251                 keybuf = &empty_keybuf;
  252 }
  253 
  254 /* It'd be nice if we could store these in some kind of secure memory... */
  255 struct keybuf * get_keybuf(void) {
  256 
  257         return (keybuf);
  258 }
  259 
  260 static int
  261 crypto_init(void)
  262 {
  263         struct crypto_ret_worker *ret_worker;
  264         int error;
  265 
  266         mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
  267                 MTX_DEF|MTX_QUIET);
  268 
  269         TAILQ_INIT(&crp_q);
  270         TAILQ_INIT(&crp_kq);
  271         mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
  272 
  273         cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
  274                                     0, 0, 0, 0,
  275                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  276         cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
  277                                     0, 0, 0, 0,
  278                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  279         cryptoses_zone = uma_zcreate("crypto_session",
  280             sizeof(struct crypto_session), NULL, NULL, NULL, NULL,
  281             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  282 
  283         if (cryptodesc_zone == NULL || cryptop_zone == NULL ||
  284             cryptoses_zone == NULL) {
  285                 printf("crypto_init: cannot setup crypto zones\n");
  286                 error = ENOMEM;
  287                 goto bad;
  288         }
  289 
  290         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
  291         crypto_drivers = malloc(crypto_drivers_num *
  292             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
  293         if (crypto_drivers == NULL) {
  294                 printf("crypto_init: cannot setup crypto drivers\n");
  295                 error = ENOMEM;
  296                 goto bad;
  297         }
  298 
  299         if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
  300                 crypto_workers_num = mp_ncpus;
  301 
  302         crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
  303                                 taskqueue_thread_enqueue, &crypto_tq);
  304         if (crypto_tq == NULL) {
  305                 printf("crypto init: cannot setup crypto taskqueue\n");
  306                 error = ENOMEM;
  307                 goto bad;
  308         }
  309 
  310         taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
  311                 "crypto");
  312 
  313         error = kproc_create((void (*)(void *)) crypto_proc, NULL,
  314                     &cryptoproc, 0, 0, "crypto");
  315         if (error) {
  316                 printf("crypto_init: cannot start crypto thread; error %d",
  317                         error);
  318                 goto bad;
  319         }
  320 
  321         crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
  322                         M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
  323         if (crypto_ret_workers == NULL) {
  324                 error = ENOMEM;
  325                 printf("crypto_init: cannot allocate ret workers\n");
  326                 goto bad;
  327         }
  328 
  329 
  330         FOREACH_CRYPTO_RETW(ret_worker) {
  331                 TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
  332                 TAILQ_INIT(&ret_worker->crp_ret_q);
  333                 TAILQ_INIT(&ret_worker->crp_ret_kq);
  334 
  335                 ret_worker->reorder_ops = 0;
  336                 ret_worker->reorder_cur_seq = 0;
  337 
  338                 mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
  339 
  340                 error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
  341                                 &ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
  342                 if (error) {
  343                         printf("crypto_init: cannot start cryptoret thread; error %d",
  344                                 error);
  345                         goto bad;
  346                 }
  347         }
  348 
  349         keybuf_init();
  350 
  351         return 0;
  352 bad:
  353         crypto_destroy();
  354         return error;
  355 }
  356 
  357 /*
  358  * Signal a crypto thread to terminate.  We use the driver
  359  * table lock to synchronize the sleep/wakeups so that we
  360  * are sure the threads have terminated before we release
  361  * the data structures they use.  See crypto_finis below
  362  * for the other half of this song-and-dance.
  363  */
  364 static void
  365 crypto_terminate(struct proc **pp, void *q)
  366 {
  367         struct proc *p;
  368 
  369         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
  370         p = *pp;
  371         *pp = NULL;
  372         if (p) {
  373                 wakeup_one(q);
  374                 PROC_LOCK(p);           /* NB: insure we don't miss wakeup */
  375                 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */
  376                 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
  377                 PROC_UNLOCK(p);
  378                 CRYPTO_DRIVER_LOCK();
  379         }
  380 }
  381 
  382 static void
  383 crypto_destroy(void)
  384 {
  385         struct crypto_ret_worker *ret_worker;
  386 
  387         /*
  388          * Terminate any crypto threads.
  389          */
  390         if (crypto_tq != NULL)
  391                 taskqueue_drain_all(crypto_tq);
  392         CRYPTO_DRIVER_LOCK();
  393         crypto_terminate(&cryptoproc, &crp_q);
  394         FOREACH_CRYPTO_RETW(ret_worker)
  395                 crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
  396         CRYPTO_DRIVER_UNLOCK();
  397 
  398         /* XXX flush queues??? */
  399 
  400         /*
  401          * Reclaim dynamically allocated resources.
  402          */
  403         if (crypto_drivers != NULL)
  404                 free(crypto_drivers, M_CRYPTO_DATA);
  405 
  406         if (cryptoses_zone != NULL)
  407                 uma_zdestroy(cryptoses_zone);
  408         if (cryptodesc_zone != NULL)
  409                 uma_zdestroy(cryptodesc_zone);
  410         if (cryptop_zone != NULL)
  411                 uma_zdestroy(cryptop_zone);
  412         mtx_destroy(&crypto_q_mtx);
  413         FOREACH_CRYPTO_RETW(ret_worker)
  414                 mtx_destroy(&ret_worker->crypto_ret_mtx);
  415         free(crypto_ret_workers, M_CRYPTO_DATA);
  416         if (crypto_tq != NULL)
  417                 taskqueue_free(crypto_tq);
  418         mtx_destroy(&crypto_drivers_mtx);
  419 }
  420 
  421 uint32_t
  422 crypto_ses2hid(crypto_session_t crypto_session)
  423 {
  424         return (crypto_session->hid);
  425 }
  426 
  427 uint32_t
  428 crypto_ses2caps(crypto_session_t crypto_session)
  429 {
  430         return (crypto_session->capabilities);
  431 }
  432 
  433 void *
  434 crypto_get_driver_session(crypto_session_t crypto_session)
  435 {
  436         return (crypto_session->softc);
  437 }
  438 
  439 static struct cryptocap *
  440 crypto_checkdriver(u_int32_t hid)
  441 {
  442         if (crypto_drivers == NULL)
  443                 return NULL;
  444         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
  445 }
  446 
  447 /*
  448  * Compare a driver's list of supported algorithms against another
  449  * list; return non-zero if all algorithms are supported.
  450  */
  451 static int
  452 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
  453 {
  454         const struct cryptoini *cr;
  455 
  456         /* See if all the algorithms are supported. */
  457         for (cr = cri; cr; cr = cr->cri_next)
  458                 if (cap->cc_alg[cr->cri_alg] == 0)
  459                         return 0;
  460         return 1;
  461 }
  462 
  463 /*
  464  * Select a driver for a new session that supports the specified
  465  * algorithms and, optionally, is constrained according to the flags.
  466  * The algorithm we use here is pretty stupid; just use the
  467  * first driver that supports all the algorithms we need. If there
  468  * are multiple drivers we choose the driver with the fewest active
  469  * sessions.  We prefer hardware-backed drivers to software ones.
  470  *
  471  * XXX We need more smarts here (in real life too, but that's
  472  * XXX another story altogether).
  473  */
  474 static struct cryptocap *
  475 crypto_select_driver(const struct cryptoini *cri, int flags)
  476 {
  477         struct cryptocap *cap, *best;
  478         int match, hid;
  479 
  480         CRYPTO_DRIVER_ASSERT();
  481 
  482         /*
  483          * Look first for hardware crypto devices if permitted.
  484          */
  485         if (flags & CRYPTOCAP_F_HARDWARE)
  486                 match = CRYPTOCAP_F_HARDWARE;
  487         else
  488                 match = CRYPTOCAP_F_SOFTWARE;
  489         best = NULL;
  490 again:
  491         for (hid = 0; hid < crypto_drivers_num; hid++) {
  492                 cap = &crypto_drivers[hid];
  493                 /*
  494                  * If it's not initialized, is in the process of
  495                  * going away, or is not appropriate (hardware
  496                  * or software based on match), then skip.
  497                  */
  498                 if (cap->cc_dev == NULL ||
  499                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
  500                     (cap->cc_flags & match) == 0)
  501                         continue;
  502 
  503                 /* verify all the algorithms are supported. */
  504                 if (driver_suitable(cap, cri)) {
  505                         if (best == NULL ||
  506                             cap->cc_sessions < best->cc_sessions)
  507                                 best = cap;
  508                 }
  509         }
  510         if (best == NULL && match == CRYPTOCAP_F_HARDWARE &&
  511             (flags & CRYPTOCAP_F_SOFTWARE)) {
  512                 /* sort of an Algol 68-style for loop */
  513                 match = CRYPTOCAP_F_SOFTWARE;
  514                 goto again;
  515         }
  516         return best;
  517 }
  518 
  519 /*
  520  * Create a new session.  The crid argument specifies a crypto
  521  * driver to use or constraints on a driver to select (hardware
  522  * only, software only, either).  Whatever driver is selected
  523  * must be capable of the requested crypto algorithms.
  524  */
  525 int
  526 crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid)
  527 {
  528         crypto_session_t res;
  529         void *softc_mem;
  530         struct cryptocap *cap;
  531         u_int32_t hid;
  532         size_t softc_size;
  533         int err;
  534 
  535 restart:
  536         res = NULL;
  537         softc_mem = NULL;
  538 
  539         CRYPTO_DRIVER_LOCK();
  540         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
  541                 /*
  542                  * Use specified driver; verify it is capable.
  543                  */
  544                 cap = crypto_checkdriver(crid);
  545                 if (cap != NULL && !driver_suitable(cap, cri))
  546                         cap = NULL;
  547         } else {
  548                 /*
  549                  * No requested driver; select based on crid flags.
  550                  */
  551                 cap = crypto_select_driver(cri, crid);
  552                 /*
  553                  * if NULL then can't do everything in one session.
  554                  * XXX Fix this. We need to inject a "virtual" session
  555                  * XXX layer right about here.
  556                  */
  557         }
  558         if (cap == NULL) {
  559                 CRYPTDEB("no driver");
  560                 err = EOPNOTSUPP;
  561                 goto out;
  562         }
  563         cap->cc_sessions++;
  564         softc_size = cap->cc_session_size;
  565         hid = cap - crypto_drivers;
  566         cap = NULL;
  567         CRYPTO_DRIVER_UNLOCK();
  568 
  569         softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO);
  570         res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
  571         res->softc = softc_mem;
  572 
  573         CRYPTO_DRIVER_LOCK();
  574         cap = crypto_checkdriver(hid);
  575         if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) {
  576                 cap->cc_sessions--;
  577                 crypto_remove(cap);
  578                 cap = NULL;
  579         }
  580         if (cap == NULL) {
  581                 free(softc_mem, M_CRYPTO_DATA);
  582                 uma_zfree(cryptoses_zone, res);
  583                 CRYPTO_DRIVER_UNLOCK();
  584                 goto restart;
  585         }
  586 
  587         /* Call the driver initialization routine. */
  588         err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri);
  589         if (err != 0) {
  590                 CRYPTDEB("dev newsession failed: %d", err);
  591                 goto out;
  592         }
  593 
  594         res->capabilities = cap->cc_flags & 0xff000000;
  595         res->hid = hid;
  596         *cses = res;
  597 
  598 out:
  599         CRYPTO_DRIVER_UNLOCK();
  600         if (err != 0) {
  601                 free(softc_mem, M_CRYPTO_DATA);
  602                 if (res != NULL)
  603                         uma_zfree(cryptoses_zone, res);
  604         }
  605         return err;
  606 }
  607 
  608 static void
  609 crypto_remove(struct cryptocap *cap)
  610 {
  611 
  612         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
  613         if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
  614                 bzero(cap, sizeof(*cap));
  615 }
  616 
  617 /*
  618  * Delete an existing session (or a reserved session on an unregistered
  619  * driver).
  620  */
  621 void
  622 crypto_freesession(crypto_session_t cses)
  623 {
  624         struct cryptocap *cap;
  625         void *ses;
  626         size_t ses_size;
  627         u_int32_t hid;
  628 
  629         if (cses == NULL)
  630                 return;
  631 
  632         CRYPTO_DRIVER_LOCK();
  633 
  634         hid = crypto_ses2hid(cses);
  635         KASSERT(hid < crypto_drivers_num,
  636             ("bogus crypto_session %p hid %u", cses, hid));
  637         cap = &crypto_drivers[hid];
  638 
  639         ses = cses->softc;
  640         ses_size = cap->cc_session_size;
  641 
  642         if (cap->cc_sessions)
  643                 cap->cc_sessions--;
  644 
  645         /* Call the driver cleanup routine, if available. */
  646         CRYPTODEV_FREESESSION(cap->cc_dev, cses);
  647 
  648         explicit_bzero(ses, ses_size);
  649         free(ses, M_CRYPTO_DATA);
  650         uma_zfree(cryptoses_zone, cses);
  651 
  652         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
  653                 crypto_remove(cap);
  654 
  655         CRYPTO_DRIVER_UNLOCK();
  656 }
  657 
  658 /*
  659  * Return an unused driver id.  Used by drivers prior to registering
  660  * support for the algorithms they handle.
  661  */
  662 int32_t
  663 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
  664 {
  665         struct cryptocap *newdrv;
  666         int i;
  667 
  668         if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
  669                 printf("%s: no flags specified when registering driver\n",
  670                     device_get_nameunit(dev));
  671                 return -1;
  672         }
  673 
  674         CRYPTO_DRIVER_LOCK();
  675 
  676         for (i = 0; i < crypto_drivers_num; i++) {
  677                 if (crypto_drivers[i].cc_dev == NULL &&
  678                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
  679                         break;
  680                 }
  681         }
  682 
  683         /* Out of entries, allocate some more. */
  684         if (i == crypto_drivers_num) {
  685                 /* Be careful about wrap-around. */
  686                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
  687                         CRYPTO_DRIVER_UNLOCK();
  688                         printf("crypto: driver count wraparound!\n");
  689                         return -1;
  690                 }
  691 
  692                 newdrv = malloc(2 * crypto_drivers_num *
  693                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
  694                 if (newdrv == NULL) {
  695                         CRYPTO_DRIVER_UNLOCK();
  696                         printf("crypto: no space to expand driver table!\n");
  697                         return -1;
  698                 }
  699 
  700                 bcopy(crypto_drivers, newdrv,
  701                     crypto_drivers_num * sizeof(struct cryptocap));
  702 
  703                 crypto_drivers_num *= 2;
  704 
  705                 free(crypto_drivers, M_CRYPTO_DATA);
  706                 crypto_drivers = newdrv;
  707         }
  708 
  709         /* NB: state is zero'd on free */
  710         crypto_drivers[i].cc_sessions = 1;      /* Mark */
  711         crypto_drivers[i].cc_dev = dev;
  712         crypto_drivers[i].cc_flags = flags;
  713         crypto_drivers[i].cc_session_size = sessionsize;
  714         if (bootverbose)
  715                 printf("crypto: assign %s driver id %u, flags 0x%x\n",
  716                     device_get_nameunit(dev), i, flags);
  717 
  718         CRYPTO_DRIVER_UNLOCK();
  719 
  720         return i;
  721 }
  722 
  723 /*
  724  * Lookup a driver by name.  We match against the full device
  725  * name and unit, and against just the name.  The latter gives
  726  * us a simple widlcarding by device name.  On success return the
  727  * driver/hardware identifier; otherwise return -1.
  728  */
  729 int
  730 crypto_find_driver(const char *match)
  731 {
  732         int i, len = strlen(match);
  733 
  734         CRYPTO_DRIVER_LOCK();
  735         for (i = 0; i < crypto_drivers_num; i++) {
  736                 device_t dev = crypto_drivers[i].cc_dev;
  737                 if (dev == NULL ||
  738                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
  739                         continue;
  740                 if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
  741                     strncmp(match, device_get_name(dev), len) == 0)
  742                         break;
  743         }
  744         CRYPTO_DRIVER_UNLOCK();
  745         return i < crypto_drivers_num ? i : -1;
  746 }
  747 
  748 /*
  749  * Return the device_t for the specified driver or NULL
  750  * if the driver identifier is invalid.
  751  */
  752 device_t
  753 crypto_find_device_byhid(int hid)
  754 {
  755         struct cryptocap *cap = crypto_checkdriver(hid);
  756         return cap != NULL ? cap->cc_dev : NULL;
  757 }
  758 
  759 /*
  760  * Return the device/driver capabilities.
  761  */
  762 int
  763 crypto_getcaps(int hid)
  764 {
  765         struct cryptocap *cap = crypto_checkdriver(hid);
  766         return cap != NULL ? cap->cc_flags : 0;
  767 }
  768 
  769 /*
  770  * Register support for a key-related algorithm.  This routine
  771  * is called once for each algorithm supported a driver.
  772  */
  773 int
  774 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
  775 {
  776         struct cryptocap *cap;
  777         int err;
  778 
  779         CRYPTO_DRIVER_LOCK();
  780 
  781         cap = crypto_checkdriver(driverid);
  782         if (cap != NULL &&
  783             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
  784                 /*
  785                  * XXX Do some performance testing to determine placing.
  786                  * XXX We probably need an auxiliary data structure that
  787                  * XXX describes relative performances.
  788                  */
  789 
  790                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
  791                 if (bootverbose)
  792                         printf("crypto: %s registers key alg %u flags %u\n"
  793                                 , device_get_nameunit(cap->cc_dev)
  794                                 , kalg
  795                                 , flags
  796                         );
  797                 err = 0;
  798         } else
  799                 err = EINVAL;
  800 
  801         CRYPTO_DRIVER_UNLOCK();
  802         return err;
  803 }
  804 
  805 /*
  806  * Register support for a non-key-related algorithm.  This routine
  807  * is called once for each such algorithm supported by a driver.
  808  */
  809 int
  810 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
  811     u_int32_t flags)
  812 {
  813         struct cryptocap *cap;
  814         int err;
  815 
  816         CRYPTO_DRIVER_LOCK();
  817 
  818         cap = crypto_checkdriver(driverid);
  819         /* NB: algorithms are in the range [1..max] */
  820         if (cap != NULL &&
  821             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
  822                 /*
  823                  * XXX Do some performance testing to determine placing.
  824                  * XXX We probably need an auxiliary data structure that
  825                  * XXX describes relative performances.
  826                  */
  827 
  828                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
  829                 cap->cc_max_op_len[alg] = maxoplen;
  830                 if (bootverbose)
  831                         printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
  832                                 , device_get_nameunit(cap->cc_dev)
  833                                 , alg
  834                                 , flags
  835                                 , maxoplen
  836                         );
  837                 cap->cc_sessions = 0;           /* Unmark */
  838                 err = 0;
  839         } else
  840                 err = EINVAL;
  841 
  842         CRYPTO_DRIVER_UNLOCK();
  843         return err;
  844 }
  845 
  846 static void
  847 driver_finis(struct cryptocap *cap)
  848 {
  849         u_int32_t ses, kops;
  850 
  851         CRYPTO_DRIVER_ASSERT();
  852 
  853         ses = cap->cc_sessions;
  854         kops = cap->cc_koperations;
  855         bzero(cap, sizeof(*cap));
  856         if (ses != 0 || kops != 0) {
  857                 /*
  858                  * If there are pending sessions,
  859                  * just mark as invalid.
  860                  */
  861                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
  862                 cap->cc_sessions = ses;
  863                 cap->cc_koperations = kops;
  864         }
  865 }
  866 
  867 /*
  868  * Unregister a crypto driver. If there are pending sessions using it,
  869  * leave enough information around so that subsequent calls using those
  870  * sessions will correctly detect the driver has been unregistered and
  871  * reroute requests.
  872  */
  873 int
  874 crypto_unregister(u_int32_t driverid, int alg)
  875 {
  876         struct cryptocap *cap;
  877         int i, err;
  878 
  879         CRYPTO_DRIVER_LOCK();
  880         cap = crypto_checkdriver(driverid);
  881         if (cap != NULL &&
  882             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
  883             cap->cc_alg[alg] != 0) {
  884                 cap->cc_alg[alg] = 0;
  885                 cap->cc_max_op_len[alg] = 0;
  886 
  887                 /* Was this the last algorithm ? */
  888                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
  889                         if (cap->cc_alg[i] != 0)
  890                                 break;
  891 
  892                 if (i == CRYPTO_ALGORITHM_MAX + 1)
  893                         driver_finis(cap);
  894                 err = 0;
  895         } else
  896                 err = EINVAL;
  897         CRYPTO_DRIVER_UNLOCK();
  898 
  899         return err;
  900 }
  901 
  902 /*
  903  * Unregister all algorithms associated with a crypto driver.
  904  * If there are pending sessions using it, leave enough information
  905  * around so that subsequent calls using those sessions will
  906  * correctly detect the driver has been unregistered and reroute
  907  * requests.
  908  */
  909 int
  910 crypto_unregister_all(u_int32_t driverid)
  911 {
  912         struct cryptocap *cap;
  913         int err;
  914 
  915         CRYPTO_DRIVER_LOCK();
  916         cap = crypto_checkdriver(driverid);
  917         if (cap != NULL) {
  918                 driver_finis(cap);
  919                 err = 0;
  920         } else
  921                 err = EINVAL;
  922         CRYPTO_DRIVER_UNLOCK();
  923 
  924         return err;
  925 }
  926 
  927 /*
  928  * Clear blockage on a driver.  The what parameter indicates whether
  929  * the driver is now ready for cryptop's and/or cryptokop's.
  930  */
  931 int
  932 crypto_unblock(u_int32_t driverid, int what)
  933 {
  934         struct cryptocap *cap;
  935         int err;
  936 
  937         CRYPTO_Q_LOCK();
  938         cap = crypto_checkdriver(driverid);
  939         if (cap != NULL) {
  940                 if (what & CRYPTO_SYMQ)
  941                         cap->cc_qblocked = 0;
  942                 if (what & CRYPTO_ASYMQ)
  943                         cap->cc_kqblocked = 0;
  944                 if (crp_sleep)
  945                         wakeup_one(&crp_q);
  946                 err = 0;
  947         } else
  948                 err = EINVAL;
  949         CRYPTO_Q_UNLOCK();
  950 
  951         return err;
  952 }
  953 
  954 /*
  955  * Add a crypto request to a queue, to be processed by the kernel thread.
  956  */
  957 int
  958 crypto_dispatch(struct cryptop *crp)
  959 {
  960         struct cryptocap *cap;
  961         u_int32_t hid;
  962         int result;
  963 
  964         cryptostats.cs_ops++;
  965 
  966 #ifdef CRYPTO_TIMING
  967         if (crypto_timing)
  968                 binuptime(&crp->crp_tstamp);
  969 #endif
  970 
  971         crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;
  972 
  973         if (CRYPTOP_ASYNC(crp)) {
  974                 if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
  975                         struct crypto_ret_worker *ret_worker;
  976 
  977                         ret_worker = CRYPTO_RETW(crp->crp_retw_id);
  978 
  979                         CRYPTO_RETW_LOCK(ret_worker);
  980                         crp->crp_seq = ret_worker->reorder_ops++;
  981                         CRYPTO_RETW_UNLOCK(ret_worker);
  982                 }
  983 
  984                 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
  985                 taskqueue_enqueue(crypto_tq, &crp->crp_task);
  986                 return (0);
  987         }
  988 
  989         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
  990                 hid = crypto_ses2hid(crp->crp_session);
  991 
  992                 /*
  993                  * Caller marked the request to be processed
  994                  * immediately; dispatch it directly to the
  995                  * driver unless the driver is currently blocked.
  996                  */
  997                 cap = crypto_checkdriver(hid);
  998                 /* Driver cannot disappeared when there is an active session. */
  999                 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
 1000                 if (!cap->cc_qblocked) {
 1001                         result = crypto_invoke(cap, crp, 0);
 1002                         if (result != ERESTART)
 1003                                 return (result);
 1004                         /*
 1005                          * The driver ran out of resources, put the request on
 1006                          * the queue.
 1007                          */
 1008                 }
 1009         }
 1010         crypto_batch_enqueue(crp);
 1011         return 0;
 1012 }
 1013 
 1014 void
 1015 crypto_batch_enqueue(struct cryptop *crp)
 1016 {
 1017 
 1018         CRYPTO_Q_LOCK();
 1019         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 1020         if (crp_sleep)
 1021                 wakeup_one(&crp_q);
 1022         CRYPTO_Q_UNLOCK();
 1023 }
 1024 
 1025 /*
 1026  * Add an asymetric crypto request to a queue,
 1027  * to be processed by the kernel thread.
 1028  */
 1029 int
 1030 crypto_kdispatch(struct cryptkop *krp)
 1031 {
 1032         int error;
 1033 
 1034         cryptostats.cs_kops++;
 1035 
 1036         error = crypto_kinvoke(krp, krp->krp_crid);
 1037         if (error == ERESTART) {
 1038                 CRYPTO_Q_LOCK();
 1039                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 1040                 if (crp_sleep)
 1041                         wakeup_one(&crp_q);
 1042                 CRYPTO_Q_UNLOCK();
 1043                 error = 0;
 1044         }
 1045         return error;
 1046 }
 1047 
 1048 /*
 1049  * Verify a driver is suitable for the specified operation.
 1050  */
 1051 static __inline int
 1052 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
 1053 {
 1054         return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
 1055 }
 1056 
 1057 /*
 1058  * Select a driver for an asym operation.  The driver must
 1059  * support the necessary algorithm.  The caller can constrain
 1060  * which device is selected with the flags parameter.  The
 1061  * algorithm we use here is pretty stupid; just use the first
 1062  * driver that supports the algorithms we need. If there are
 1063  * multiple suitable drivers we choose the driver with the
 1064  * fewest active operations.  We prefer hardware-backed
 1065  * drivers to software ones when either may be used.
 1066  */
 1067 static struct cryptocap *
 1068 crypto_select_kdriver(const struct cryptkop *krp, int flags)
 1069 {
 1070         struct cryptocap *cap, *best;
 1071         int match, hid;
 1072 
 1073         CRYPTO_DRIVER_ASSERT();
 1074 
 1075         /*
 1076          * Look first for hardware crypto devices if permitted.
 1077          */
 1078         if (flags & CRYPTOCAP_F_HARDWARE)
 1079                 match = CRYPTOCAP_F_HARDWARE;
 1080         else
 1081                 match = CRYPTOCAP_F_SOFTWARE;
 1082         best = NULL;
 1083 again:
 1084         for (hid = 0; hid < crypto_drivers_num; hid++) {
 1085                 cap = &crypto_drivers[hid];
 1086                 /*
 1087                  * If it's not initialized, is in the process of
 1088                  * going away, or is not appropriate (hardware
 1089                  * or software based on match), then skip.
 1090                  */
 1091                 if (cap->cc_dev == NULL ||
 1092                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
 1093                     (cap->cc_flags & match) == 0)
 1094                         continue;
 1095 
 1096                 /* verify all the algorithms are supported. */
 1097                 if (kdriver_suitable(cap, krp)) {
 1098                         if (best == NULL ||
 1099                             cap->cc_koperations < best->cc_koperations)
 1100                                 best = cap;
 1101                 }
 1102         }
 1103         if (best != NULL)
 1104                 return best;
 1105         if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
 1106                 /* sort of an Algol 68-style for loop */
 1107                 match = CRYPTOCAP_F_SOFTWARE;
 1108                 goto again;
 1109         }
 1110         return best;
 1111 }
 1112 
 1113 /*
 1114  * Dispatch an asymmetric crypto request.
 1115  */
 1116 static int
 1117 crypto_kinvoke(struct cryptkop *krp, int crid)
 1118 {
 1119         struct cryptocap *cap = NULL;
 1120         int error;
 1121 
 1122         KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
 1123         KASSERT(krp->krp_callback != NULL,
 1124             ("%s: krp->crp_callback == NULL", __func__));
 1125 
 1126         CRYPTO_DRIVER_LOCK();
 1127         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 1128                 cap = crypto_checkdriver(crid);
 1129                 if (cap != NULL) {
 1130                         /*
 1131                          * Driver present, it must support the necessary
 1132                          * algorithm and, if s/w drivers are excluded,
 1133                          * it must be registered as hardware-backed.
 1134                          */
 1135                         if (!kdriver_suitable(cap, krp) ||
 1136                             (!crypto_devallowsoft &&
 1137                              (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
 1138                                 cap = NULL;
 1139                 }
 1140         } else {
 1141                 /*
 1142                  * No requested driver; select based on crid flags.
 1143                  */
 1144                 if (!crypto_devallowsoft)       /* NB: disallow s/w drivers */
 1145                         crid &= ~CRYPTOCAP_F_SOFTWARE;
 1146                 cap = crypto_select_kdriver(krp, crid);
 1147         }
 1148         if (cap != NULL && !cap->cc_kqblocked) {
 1149                 krp->krp_hid = cap - crypto_drivers;
 1150                 cap->cc_koperations++;
 1151                 CRYPTO_DRIVER_UNLOCK();
 1152                 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
 1153                 CRYPTO_DRIVER_LOCK();
 1154                 if (error == ERESTART) {
 1155                         cap->cc_koperations--;
 1156                         CRYPTO_DRIVER_UNLOCK();
 1157                         return (error);
 1158                 }
 1159         } else {
 1160                 /*
 1161                  * NB: cap is !NULL if device is blocked; in
 1162                  *     that case return ERESTART so the operation
 1163                  *     is resubmitted if possible.
 1164                  */
 1165                 error = (cap == NULL) ? ENODEV : ERESTART;
 1166         }
 1167         CRYPTO_DRIVER_UNLOCK();
 1168 
 1169         if (error) {
 1170                 krp->krp_status = error;
 1171                 crypto_kdone(krp);
 1172         }
 1173         return 0;
 1174 }
 1175 
 1176 #ifdef CRYPTO_TIMING
 1177 static void
 1178 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
 1179 {
 1180         struct bintime now, delta;
 1181         struct timespec t;
 1182         uint64_t u;
 1183 
 1184         binuptime(&now);
 1185         u = now.frac;
 1186         delta.frac = now.frac - bt->frac;
 1187         delta.sec = now.sec - bt->sec;
 1188         if (u < delta.frac)
 1189                 delta.sec--;
 1190         bintime2timespec(&delta, &t);
 1191         timespecadd(&ts->acc, &t, &ts->acc);
 1192         if (timespeccmp(&t, &ts->min, <))
 1193                 ts->min = t;
 1194         if (timespeccmp(&t, &ts->max, >))
 1195                 ts->max = t;
 1196         ts->count++;
 1197 
 1198         *bt = now;
 1199 }
 1200 #endif
 1201 
 1202 static void
 1203 crypto_task_invoke(void *ctx, int pending)
 1204 {
 1205         struct cryptocap *cap;
 1206         struct cryptop *crp;
 1207         int hid, result;
 1208 
 1209         crp = (struct cryptop *)ctx;
 1210 
 1211         hid = crypto_ses2hid(crp->crp_session);
 1212         cap = crypto_checkdriver(hid);
 1213 
 1214         result = crypto_invoke(cap, crp, 0);
 1215         if (result == ERESTART)
 1216                 crypto_batch_enqueue(crp);
 1217 }
 1218 
 1219 /*
 1220  * Dispatch a crypto request to the appropriate crypto devices.
 1221  */
 1222 static int
 1223 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
 1224 {
 1225 
 1226         KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
 1227         KASSERT(crp->crp_callback != NULL,
 1228             ("%s: crp->crp_callback == NULL", __func__));
 1229         KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
 1230 
 1231 #ifdef CRYPTO_TIMING
 1232         if (crypto_timing)
 1233                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
 1234 #endif
 1235         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
 1236                 struct cryptodesc *crd;
 1237                 crypto_session_t nses;
 1238 
 1239                 /*
 1240                  * Driver has unregistered; migrate the session and return
 1241                  * an error to the caller so they'll resubmit the op.
 1242                  *
 1243                  * XXX: What if there are more already queued requests for this
 1244                  *      session?
 1245                  */
 1246                 crypto_freesession(crp->crp_session);
 1247 
 1248                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
 1249                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
 1250 
 1251                 /* XXX propagate flags from initial session? */
 1252                 if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI),
 1253                     CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
 1254                         crp->crp_session = nses;
 1255 
 1256                 crp->crp_etype = EAGAIN;
 1257                 crypto_done(crp);
 1258                 return 0;
 1259         } else {
 1260                 /*
 1261                  * Invoke the driver to process the request.
 1262                  */
 1263                 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
 1264         }
 1265 }
 1266 
 1267 /*
 1268  * Release a set of crypto descriptors.
 1269  */
 1270 void
 1271 crypto_freereq(struct cryptop *crp)
 1272 {
 1273         struct cryptodesc *crd;
 1274 
 1275         if (crp == NULL)
 1276                 return;
 1277 
 1278 #ifdef DIAGNOSTIC
 1279         {
 1280                 struct cryptop *crp2;
 1281                 struct crypto_ret_worker *ret_worker;
 1282 
 1283                 CRYPTO_Q_LOCK();
 1284                 TAILQ_FOREACH(crp2, &crp_q, crp_next) {
 1285                         KASSERT(crp2 != crp,
 1286                             ("Freeing cryptop from the crypto queue (%p).",
 1287                             crp));
 1288                 }
 1289                 CRYPTO_Q_UNLOCK();
 1290 
 1291                 FOREACH_CRYPTO_RETW(ret_worker) {
 1292                         CRYPTO_RETW_LOCK(ret_worker);
 1293                         TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
 1294                                 KASSERT(crp2 != crp,
 1295                                     ("Freeing cryptop from the return queue (%p).",
 1296                                     crp));
 1297                         }
 1298                         CRYPTO_RETW_UNLOCK(ret_worker);
 1299                 }
 1300         }
 1301 #endif
 1302 
 1303         while ((crd = crp->crp_desc) != NULL) {
 1304                 crp->crp_desc = crd->crd_next;
 1305                 uma_zfree(cryptodesc_zone, crd);
 1306         }
 1307         uma_zfree(cryptop_zone, crp);
 1308 }
 1309 
 1310 /*
 1311  * Acquire a set of crypto descriptors.
 1312  */
 1313 struct cryptop *
 1314 crypto_getreq(int num)
 1315 {
 1316         struct cryptodesc *crd;
 1317         struct cryptop *crp;
 1318 
 1319         crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
 1320         if (crp != NULL) {
 1321                 while (num--) {
 1322                         crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
 1323                         if (crd == NULL) {
 1324                                 crypto_freereq(crp);
 1325                                 return NULL;
 1326                         }
 1327 
 1328                         crd->crd_next = crp->crp_desc;
 1329                         crp->crp_desc = crd;
 1330                 }
 1331         }
 1332         return crp;
 1333 }
 1334 
 1335 /*
 1336  * Invoke the callback on behalf of the driver.
 1337  */
 1338 void
 1339 crypto_done(struct cryptop *crp)
 1340 {
 1341         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
 1342                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
 1343         crp->crp_flags |= CRYPTO_F_DONE;
 1344         if (crp->crp_etype != 0)
 1345                 cryptostats.cs_errs++;
 1346 #ifdef CRYPTO_TIMING
 1347         if (crypto_timing)
 1348                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 1349 #endif
 1350         /*
 1351          * CBIMM means unconditionally do the callback immediately;
 1352          * CBIFSYNC means do the callback immediately only if the
 1353          * operation was done synchronously.  Both are used to avoid
 1354          * doing extraneous context switches; the latter is mostly
 1355          * used with the software crypto driver.
 1356          */
 1357         if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
 1358             ((crp->crp_flags & CRYPTO_F_CBIMM) ||
 1359             ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
 1360              (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) {
 1361                 /*
 1362                  * Do the callback directly.  This is ok when the
 1363                  * callback routine does very little (e.g. the
 1364                  * /dev/crypto callback method just does a wakeup).
 1365                  */
 1366 #ifdef CRYPTO_TIMING
 1367                 if (crypto_timing) {
 1368                         /*
 1369                          * NB: We must copy the timestamp before
 1370                          * doing the callback as the cryptop is
 1371                          * likely to be reclaimed.
 1372                          */
 1373                         struct bintime t = crp->crp_tstamp;
 1374                         crypto_tstat(&cryptostats.cs_cb, &t);
 1375                         crp->crp_callback(crp);
 1376                         crypto_tstat(&cryptostats.cs_finis, &t);
 1377                 } else
 1378 #endif
 1379                         crp->crp_callback(crp);
 1380         } else {
 1381                 struct crypto_ret_worker *ret_worker;
 1382                 bool wake;
 1383 
 1384                 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
 1385                 wake = false;
 1386 
 1387                 /*
 1388                  * Normal case; queue the callback for the thread.
 1389                  */
 1390                 CRYPTO_RETW_LOCK(ret_worker);
 1391                 if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
 1392                         struct cryptop *tmp;
 1393 
 1394                         TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
 1395                                         cryptop_q, crp_next) {
 1396                                 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
 1397                                         TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
 1398                                                         tmp, crp, crp_next);
 1399                                         break;
 1400                                 }
 1401                         }
 1402                         if (tmp == NULL) {
 1403                                 TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
 1404                                                 crp, crp_next);
 1405                         }
 1406 
 1407                         if (crp->crp_seq == ret_worker->reorder_cur_seq)
 1408                                 wake = true;
 1409                 }
 1410                 else {
 1411                         if (CRYPTO_RETW_EMPTY(ret_worker))
 1412                                 wake = true;
 1413 
 1414                         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
 1415                 }
 1416 
 1417                 if (wake)
 1418                         wakeup_one(&ret_worker->crp_ret_q);     /* shared wait channel */
 1419                 CRYPTO_RETW_UNLOCK(ret_worker);
 1420         }
 1421 }
 1422 
 1423 /*
 1424  * Invoke the callback on behalf of the driver.
 1425  */
 1426 void
 1427 crypto_kdone(struct cryptkop *krp)
 1428 {
 1429         struct crypto_ret_worker *ret_worker;
 1430         struct cryptocap *cap;
 1431 
 1432         if (krp->krp_status != 0)
 1433                 cryptostats.cs_kerrs++;
 1434         CRYPTO_DRIVER_LOCK();
 1435         /* XXX: What if driver is loaded in the meantime? */
 1436         if (krp->krp_hid < crypto_drivers_num) {
 1437                 cap = &crypto_drivers[krp->krp_hid];
 1438                 KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
 1439                 cap->cc_koperations--;
 1440                 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
 1441                         crypto_remove(cap);
 1442         }
 1443         CRYPTO_DRIVER_UNLOCK();
 1444 
 1445         ret_worker = CRYPTO_RETW(0);
 1446 
 1447         CRYPTO_RETW_LOCK(ret_worker);
 1448         if (CRYPTO_RETW_EMPTY(ret_worker))
 1449                 wakeup_one(&ret_worker->crp_ret_q);             /* shared wait channel */
 1450         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
 1451         CRYPTO_RETW_UNLOCK(ret_worker);
 1452 }
 1453 
 1454 int
 1455 crypto_getfeat(int *featp)
 1456 {
 1457         int hid, kalg, feat = 0;
 1458 
 1459         CRYPTO_DRIVER_LOCK();
 1460         for (hid = 0; hid < crypto_drivers_num; hid++) {
 1461                 const struct cryptocap *cap = &crypto_drivers[hid];
 1462 
 1463                 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
 1464                     !crypto_devallowsoft) {
 1465                         continue;
 1466                 }
 1467                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
 1468                         if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
 1469                                 feat |=  1 << kalg;
 1470         }
 1471         CRYPTO_DRIVER_UNLOCK();
 1472         *featp = feat;
 1473         return (0);
 1474 }
 1475 
 1476 /*
 1477  * Terminate a thread at module unload.  The process that
 1478  * initiated this is waiting for us to signal that we're gone;
 1479  * wake it up and exit.  We use the driver table lock to insure
 1480  * we don't do the wakeup before they're waiting.  There is no
 1481  * race here because the waiter sleeps on the proc lock for the
 1482  * thread so it gets notified at the right time because of an
 1483  * extra wakeup that's done in exit1().
 1484  */
 1485 static void
 1486 crypto_finis(void *chan)
 1487 {
 1488         CRYPTO_DRIVER_LOCK();
 1489         wakeup_one(chan);
 1490         CRYPTO_DRIVER_UNLOCK();
 1491         kproc_exit(0);
 1492 }
 1493 
 1494 /*
 1495  * Crypto thread, dispatches crypto requests.
 1496  */
 1497 static void
 1498 crypto_proc(void)
 1499 {
 1500         struct cryptop *crp, *submit;
 1501         struct cryptkop *krp;
 1502         struct cryptocap *cap;
 1503         u_int32_t hid;
 1504         int result, hint;
 1505 
 1506 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
 1507         fpu_kern_thread(FPU_KERN_NORMAL);
 1508 #endif
 1509 
 1510         CRYPTO_Q_LOCK();
 1511         for (;;) {
 1512                 /*
 1513                  * Find the first element in the queue that can be
 1514                  * processed and look-ahead to see if multiple ops
 1515                  * are ready for the same driver.
 1516                  */
 1517                 submit = NULL;
 1518                 hint = 0;
 1519                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
 1520                         hid = crypto_ses2hid(crp->crp_session);
 1521                         cap = crypto_checkdriver(hid);
 1522                         /*
 1523                          * Driver cannot disappeared when there is an active
 1524                          * session.
 1525                          */
 1526                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
 1527                             __func__, __LINE__));
 1528                         if (cap == NULL || cap->cc_dev == NULL) {
 1529                                 /* Op needs to be migrated, process it. */
 1530                                 if (submit == NULL)
 1531                                         submit = crp;
 1532                                 break;
 1533                         }
 1534                         if (!cap->cc_qblocked) {
 1535                                 if (submit != NULL) {
 1536                                         /*
 1537                                          * We stop on finding another op,
 1538                                          * regardless whether its for the same
 1539                                          * driver or not.  We could keep
 1540                                          * searching the queue but it might be
 1541                                          * better to just use a per-driver
 1542                                          * queue instead.
 1543                                          */
 1544                                         if (crypto_ses2hid(submit->crp_session) == hid)
 1545                                                 hint = CRYPTO_HINT_MORE;
 1546                                         break;
 1547                                 } else {
 1548                                         submit = crp;
 1549                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
 1550                                                 break;
 1551                                         /* keep scanning for more are q'd */
 1552                                 }
 1553                         }
 1554                 }
 1555                 if (submit != NULL) {
 1556                         TAILQ_REMOVE(&crp_q, submit, crp_next);
 1557                         hid = crypto_ses2hid(submit->crp_session);
 1558                         cap = crypto_checkdriver(hid);
 1559                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
 1560                             __func__, __LINE__));
 1561                         result = crypto_invoke(cap, submit, hint);
 1562                         if (result == ERESTART) {
 1563                                 /*
 1564                                  * The driver ran out of resources, mark the
 1565                                  * driver ``blocked'' for cryptop's and put
 1566                                  * the request back in the queue.  It would
 1567                                  * best to put the request back where we got
 1568                                  * it but that's hard so for now we put it
 1569                                  * at the front.  This should be ok; putting
 1570                                  * it at the end does not work.
 1571                                  */
 1572                                 /* XXX validate sid again? */
 1573                                 crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1;
 1574                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
 1575                                 cryptostats.cs_blocks++;
 1576                         }
 1577                 }
 1578 
 1579                 /* As above, but for key ops */
 1580                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
 1581                         cap = crypto_checkdriver(krp->krp_hid);
 1582                         if (cap == NULL || cap->cc_dev == NULL) {
 1583                                 /*
 1584                                  * Operation needs to be migrated, invalidate
 1585                                  * the assigned device so it will reselect a
 1586                                  * new one below.  Propagate the original
 1587                                  * crid selection flags if supplied.
 1588                                  */
 1589                                 krp->krp_hid = krp->krp_crid &
 1590                                     (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
 1591                                 if (krp->krp_hid == 0)
 1592                                         krp->krp_hid =
 1593                                     CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
 1594                                 break;
 1595                         }
 1596                         if (!cap->cc_kqblocked)
 1597                                 break;
 1598                 }
 1599                 if (krp != NULL) {
 1600                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
 1601                         result = crypto_kinvoke(krp, krp->krp_hid);
 1602                         if (result == ERESTART) {
 1603                                 /*
 1604                                  * The driver ran out of resources, mark the
 1605                                  * driver ``blocked'' for cryptkop's and put
 1606                                  * the request back in the queue.  It would
 1607                                  * best to put the request back where we got
 1608                                  * it but that's hard so for now we put it
 1609                                  * at the front.  This should be ok; putting
 1610                                  * it at the end does not work.
 1611                                  */
 1612                                 /* XXX validate sid again? */
 1613                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
 1614                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
 1615                                 cryptostats.cs_kblocks++;
 1616                         }
 1617                 }
 1618 
 1619                 if (submit == NULL && krp == NULL) {
 1620                         /*
 1621                          * Nothing more to be processed.  Sleep until we're
 1622                          * woken because there are more ops to process.
 1623                          * This happens either by submission or by a driver
 1624                          * becoming unblocked and notifying us through
 1625                          * crypto_unblock.  Note that when we wakeup we
 1626                          * start processing each queue again from the
 1627                          * front. It's not clear that it's important to
 1628                          * preserve this ordering since ops may finish
 1629                          * out of order if dispatched to different devices
 1630                          * and some become blocked while others do not.
 1631                          */
 1632                         crp_sleep = 1;
 1633                         msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
 1634                         crp_sleep = 0;
 1635                         if (cryptoproc == NULL)
 1636                                 break;
 1637                         cryptostats.cs_intrs++;
 1638                 }
 1639         }
 1640         CRYPTO_Q_UNLOCK();
 1641 
 1642         crypto_finis(&crp_q);
 1643 }
 1644 
 1645 /*
 1646  * Crypto returns thread, does callbacks for processed crypto requests.
 1647  * Callbacks are done here, rather than in the crypto drivers, because
 1648  * callbacks typically are expensive and would slow interrupt handling.
 1649  */
 1650 static void
 1651 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
 1652 {
 1653         struct cryptop *crpt;
 1654         struct cryptkop *krpt;
 1655 
 1656         CRYPTO_RETW_LOCK(ret_worker);
 1657         for (;;) {
 1658                 /* Harvest return q's for completed ops */
 1659                 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
 1660                 if (crpt != NULL) {
 1661                         if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
 1662                                 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
 1663                                 ret_worker->reorder_cur_seq++;
 1664                         } else {
 1665                                 crpt = NULL;
 1666                         }
 1667                 }
 1668 
 1669                 if (crpt == NULL) {
 1670                         crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
 1671                         if (crpt != NULL)
 1672                                 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
 1673                 }
 1674 
 1675                 krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
 1676                 if (krpt != NULL)
 1677                         TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
 1678 
 1679                 if (crpt != NULL || krpt != NULL) {
 1680                         CRYPTO_RETW_UNLOCK(ret_worker);
 1681                         /*
 1682                          * Run callbacks unlocked.
 1683                          */
 1684                         if (crpt != NULL) {
 1685 #ifdef CRYPTO_TIMING
 1686                                 if (crypto_timing) {
 1687                                         /*
 1688                                          * NB: We must copy the timestamp before
 1689                                          * doing the callback as the cryptop is
 1690                                          * likely to be reclaimed.
 1691                                          */
 1692                                         struct bintime t = crpt->crp_tstamp;
 1693                                         crypto_tstat(&cryptostats.cs_cb, &t);
 1694                                         crpt->crp_callback(crpt);
 1695                                         crypto_tstat(&cryptostats.cs_finis, &t);
 1696                                 } else
 1697 #endif
 1698                                         crpt->crp_callback(crpt);
 1699                         }
 1700                         if (krpt != NULL)
 1701                                 krpt->krp_callback(krpt);
 1702                         CRYPTO_RETW_LOCK(ret_worker);
 1703                 } else {
 1704                         /*
 1705                          * Nothing more to be processed.  Sleep until we're
 1706                          * woken because there are more returns to process.
 1707                          */
 1708                         msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
 1709                                 "crypto_ret_wait", 0);
 1710                         if (ret_worker->cryptoretproc == NULL)
 1711                                 break;
 1712                         cryptostats.cs_rets++;
 1713                 }
 1714         }
 1715         CRYPTO_RETW_UNLOCK(ret_worker);
 1716 
 1717         crypto_finis(&ret_worker->crp_ret_q);
 1718 }
 1719 
 1720 #ifdef DDB
 1721 static void
 1722 db_show_drivers(void)
 1723 {
 1724         int hid;
 1725 
 1726         db_printf("%12s %4s %4s %8s %2s %2s\n"
 1727                 , "Device"
 1728                 , "Ses"
 1729                 , "Kops"
 1730                 , "Flags"
 1731                 , "QB"
 1732                 , "KB"
 1733         );
 1734         for (hid = 0; hid < crypto_drivers_num; hid++) {
 1735                 const struct cryptocap *cap = &crypto_drivers[hid];
 1736                 if (cap->cc_dev == NULL)
 1737                         continue;
 1738                 db_printf("%-12s %4u %4u %08x %2u %2u\n"
 1739                     , device_get_nameunit(cap->cc_dev)
 1740                     , cap->cc_sessions
 1741                     , cap->cc_koperations
 1742                     , cap->cc_flags
 1743                     , cap->cc_qblocked
 1744                     , cap->cc_kqblocked
 1745                 );
 1746         }
 1747 }
 1748 
 1749 DB_SHOW_COMMAND(crypto, db_show_crypto)
 1750 {
 1751         struct cryptop *crp;
 1752         struct crypto_ret_worker *ret_worker;
 1753 
 1754         db_show_drivers();
 1755         db_printf("\n");
 1756 
 1757         db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
 1758             "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
 1759             "Desc", "Callback");
 1760         TAILQ_FOREACH(crp, &crp_q, crp_next) {
 1761                 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
 1762                     , (int) crypto_ses2hid(crp->crp_session)
 1763                     , (int) crypto_ses2caps(crp->crp_session)
 1764                     , crp->crp_ilen, crp->crp_olen
 1765                     , crp->crp_etype
 1766                     , crp->crp_flags
 1767                     , crp->crp_desc
 1768                     , crp->crp_callback
 1769                 );
 1770         }
 1771         FOREACH_CRYPTO_RETW(ret_worker) {
 1772                 db_printf("\n%8s %4s %4s %4s %8s\n",
 1773                     "ret_worker", "HID", "Etype", "Flags", "Callback");
 1774                 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
 1775                         TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
 1776                                 db_printf("%8td %4u %4u %04x %8p\n"
 1777                                     , CRYPTO_RETW_ID(ret_worker)
 1778                                     , (int) crypto_ses2hid(crp->crp_session)
 1779                                     , crp->crp_etype
 1780                                     , crp->crp_flags
 1781                                     , crp->crp_callback
 1782                                 );
 1783                         }
 1784                 }
 1785         }
 1786 }
 1787 
 1788 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
 1789 {
 1790         struct cryptkop *krp;
 1791         struct crypto_ret_worker *ret_worker;
 1792 
 1793         db_show_drivers();
 1794         db_printf("\n");
 1795 
 1796         db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
 1797             "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
 1798         TAILQ_FOREACH(krp, &crp_kq, krp_next) {
 1799                 db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
 1800                     , krp->krp_op
 1801                     , krp->krp_status
 1802                     , krp->krp_iparams, krp->krp_oparams
 1803                     , krp->krp_crid, krp->krp_hid
 1804                     , krp->krp_callback
 1805                 );
 1806         }
 1807 
 1808         ret_worker = CRYPTO_RETW(0);
 1809         if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
 1810                 db_printf("%4s %5s %8s %4s %8s\n",
 1811                     "Op", "Status", "CRID", "HID", "Callback");
 1812                 TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
 1813                         db_printf("%4u %5u %08x %4u %8p\n"
 1814                             , krp->krp_op
 1815                             , krp->krp_status
 1816                             , krp->krp_crid, krp->krp_hid
 1817                             , krp->krp_callback
 1818                         );
 1819                 }
 1820         }
 1821 }
 1822 #endif
 1823 
 1824 int crypto_modevent(module_t mod, int type, void *unused);
 1825 
 1826 /*
 1827  * Initialization code, both for static and dynamic loading.
 1828  * Note this is not invoked with the usual MODULE_DECLARE
 1829  * mechanism but instead is listed as a dependency by the
 1830  * cryptosoft driver.  This guarantees proper ordering of
 1831  * calls on module load/unload.
 1832  */
 1833 int
 1834 crypto_modevent(module_t mod, int type, void *unused)
 1835 {
 1836         int error = EINVAL;
 1837 
 1838         switch (type) {
 1839         case MOD_LOAD:
 1840                 error = crypto_init();
 1841                 if (error == 0 && bootverbose)
 1842                         printf("crypto: <crypto core>\n");
 1843                 break;
 1844         case MOD_UNLOAD:
 1845                 /*XXX disallow if active sessions */
 1846                 error = 0;
 1847                 crypto_destroy();
 1848                 return 0;
 1849         }
 1850         return error;
 1851 }
 1852 MODULE_VERSION(crypto, 1);
 1853 MODULE_DEPEND(crypto, zlib, 1, 1, 1);

Cache object: a6766e783edf8bdcc98fa4f2a14699d5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.