Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 */ 24 25 #include <sys/cdefs.h> 26 __FBSDID("$FreeBSD$"); 27 28 /* 29 * Cryptographic Subsystem. 30 * 31 * This code is derived from the Openbsd Cryptographic Framework (OCF) 32 * that has the copyright shown below. Very little of the original 33 * code remains. 34 */ 35 36 /*- 37 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 38 * 39 * This code was written by Angelos D. Keromytis in Athens, Greece, in 40 * February 2000. Network Security Technologies Inc. (NSTI) kindly 41 * supported the development of this code. 42 * 43 * Copyright (c) 2000, 2001 Angelos D. Keromytis 44 * 45 * Permission to use, copy, and modify this software with or without fee 46 * is hereby granted, provided that this entire notice is included in 47 * all source code copies of any software which is or includes a copy or 48 * modification of this software. 49 * 50 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 51 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 52 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 53 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 54 * PURPOSE. 55 */ 56 57 #include "opt_ddb.h" 58 59 #include <sys/param.h> 60 #include <sys/systm.h> 61 #include <sys/counter.h> 62 #include <sys/kernel.h> 63 #include <sys/kthread.h> 64 #include <sys/linker.h> 65 #include <sys/lock.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/malloc.h> 69 #include <sys/proc.h> 70 #include <sys/sdt.h> 71 #include <sys/smp.h> 72 #include <sys/sysctl.h> 73 #include <sys/taskqueue.h> 74 75 #include <ddb/ddb.h> 76 77 #include <vm/uma.h> 78 #include <crypto/intake.h> 79 #include <opencrypto/cryptodev.h> 80 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 81 82 #include <sys/kobj.h> 83 #include <sys/bus.h> 84 #include "cryptodev_if.h" 85 86 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 87 #include <machine/pcb.h> 88 #endif 89 90 struct crypto_session { 91 device_t parent; 92 void *softc; 93 uint32_t hid; 94 uint32_t capabilities; 95 }; 96 97 SDT_PROVIDER_DEFINE(opencrypto); 98 99 /* 100 * Crypto drivers register themselves by allocating a slot in the 101 * crypto_drivers table with crypto_get_driverid() and then registering 102 * each algorithm they support with crypto_register() and crypto_kregister(). 103 */ 104 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 105 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 106 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 107 #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED) 108 109 /* 110 * Crypto device/driver capabilities structure. 111 * 112 * Synchronization: 113 * (d) - protected by CRYPTO_DRIVER_LOCK() 114 * (q) - protected by CRYPTO_Q_LOCK() 115 * Not tagged fields are read-only. 116 */ 117 struct cryptocap { 118 device_t cc_dev; /* (d) device/driver */ 119 u_int32_t cc_sessions; /* (d) # of sessions */ 120 u_int32_t cc_koperations; /* (d) # os asym operations */ 121 /* 122 * Largest possible operator length (in bits) for each type of 123 * encryption algorithm. XXX not used 124 */ 125 u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; 126 u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; 127 u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; 128 129 int cc_flags; /* (d) flags */ 130 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ 131 int cc_qblocked; /* (q) symmetric q blocked */ 132 int cc_kqblocked; /* (q) asymmetric q blocked */ 133 size_t cc_session_size; 134 }; 135 static struct cryptocap *crypto_drivers = NULL; 136 static int crypto_drivers_num = 0; 137 138 /* 139 * There are two queues for crypto requests; one for symmetric (e.g. 140 * cipher) operations and one for asymmetric (e.g. MOD)operations. 141 * A single mutex is used to lock access to both queues. We could 142 * have one per-queue but having one simplifies handling of block/unblock 143 * operations. 144 */ 145 static int crp_sleep = 0; 146 static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */ 147 static TAILQ_HEAD(,cryptkop) crp_kq; 148 static struct mtx crypto_q_mtx; 149 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 150 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 151 152 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0, 153 "In-kernel cryptography"); 154 155 /* 156 * Taskqueue used to dispatch the crypto requests 157 * that have the CRYPTO_F_ASYNC flag 158 */ 159 static struct taskqueue *crypto_tq; 160 161 /* 162 * Crypto seq numbers are operated on with modular arithmetic 163 */ 164 #define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0) 165 166 struct crypto_ret_worker { 167 struct mtx crypto_ret_mtx; 168 169 TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */ 170 TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */ 171 TAILQ_HEAD(,cryptkop) crp_ret_kq; /* callback queue for asym jobs */ 172 173 u_int32_t reorder_ops; /* total ordered sym jobs received */ 174 u_int32_t reorder_cur_seq; /* current sym job dispatched */ 175 176 struct proc *cryptoretproc; 177 }; 178 static struct crypto_ret_worker *crypto_ret_workers = NULL; 179 180 #define CRYPTO_RETW(i) (&crypto_ret_workers[i]) 181 #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers) 182 #define FOREACH_CRYPTO_RETW(w) \ 183 for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) 184 185 #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) 186 #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) 187 #define CRYPTO_RETW_EMPTY(w) \ 188 (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q)) 189 190 static int crypto_workers_num = 0; 191 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, 192 &crypto_workers_num, 0, 193 "Number of crypto workers used to dispatch crypto jobs"); 194 195 static uma_zone_t cryptop_zone; 196 static uma_zone_t cryptodesc_zone; 197 static uma_zone_t cryptoses_zone; 198 199 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 200 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 201 &crypto_userasymcrypto, 0, 202 "Enable/disable user-mode access to asymmetric crypto support"); 203 int crypto_devallowsoft = 0; /* only use hardware crypto */ 204 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 205 &crypto_devallowsoft, 0, 206 "Enable/disable use of software crypto by /dev/crypto"); 207 208 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 209 210 static void crypto_proc(void); 211 static struct proc *cryptoproc; 212 static void crypto_ret_proc(struct crypto_ret_worker *ret_worker); 213 static void crypto_destroy(void); 214 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 215 static int crypto_kinvoke(struct cryptkop *krp, int flags); 216 static void crypto_remove(struct cryptocap *cap); 217 static void crypto_task_invoke(void *ctx, int pending); 218 static void crypto_batch_enqueue(struct cryptop *crp); 219 220 static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)]; 221 SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, 222 cryptostats, nitems(cryptostats), 223 "Crypto system statistics"); 224 225 #define CRYPTOSTAT_INC(stat) do { \ 226 counter_u64_add( \ 227 cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\ 228 1); \ 229 } while (0) 230 231 static void 232 cryptostats_init(void *arg __unused) 233 { 234 COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK); 235 } 236 SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL); 237 238 static void 239 cryptostats_fini(void *arg __unused) 240 { 241 COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats)); 242 } 243 SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini, 244 NULL); 245 246 /* Try to avoid directly exposing the key buffer as a symbol */ 247 static struct keybuf *keybuf; 248 249 static struct keybuf empty_keybuf = { 250 .kb_nents = 0 251 }; 252 253 /* Obtain the key buffer from boot metadata */ 254 static void 255 keybuf_init(void) 256 { 257 caddr_t kmdp; 258 259 kmdp = preload_search_by_type("elf kernel"); 260 261 if (kmdp == NULL) 262 kmdp = preload_search_by_type("elf64 kernel"); 263 264 keybuf = (struct keybuf *)preload_search_info(kmdp, 265 MODINFO_METADATA | MODINFOMD_KEYBUF); 266 267 if (keybuf == NULL) 268 keybuf = &empty_keybuf; 269 } 270 271 /* It'd be nice if we could store these in some kind of secure memory... */ 272 struct keybuf * get_keybuf(void) { 273 274 return (keybuf); 275 } 276 277 static int 278 crypto_init(void) 279 { 280 struct crypto_ret_worker *ret_worker; 281 int error; 282 283 mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 284 MTX_DEF|MTX_QUIET); 285 286 TAILQ_INIT(&crp_q); 287 TAILQ_INIT(&crp_kq); 288 mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 289 290 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 291 0, 0, 0, 0, 292 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 293 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 294 0, 0, 0, 0, 295 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 296 cryptoses_zone = uma_zcreate("crypto_session", 297 sizeof(struct crypto_session), NULL, NULL, NULL, NULL, 298 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 299 300 if (cryptodesc_zone == NULL || cryptop_zone == NULL || 301 cryptoses_zone == NULL) { 302 printf("crypto_init: cannot setup crypto zones\n"); 303 error = ENOMEM; 304 goto bad; 305 } 306 307 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 308 crypto_drivers = malloc(crypto_drivers_num * 309 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 310 if (crypto_drivers == NULL) { 311 printf("crypto_init: cannot setup crypto drivers\n"); 312 error = ENOMEM; 313 goto bad; 314 } 315 316 if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus) 317 crypto_workers_num = mp_ncpus; 318 319 crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO, 320 taskqueue_thread_enqueue, &crypto_tq); 321 if (crypto_tq == NULL) { 322 printf("crypto init: cannot setup crypto taskqueue\n"); 323 error = ENOMEM; 324 goto bad; 325 } 326 327 taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN, 328 "crypto"); 329 330 error = kproc_create((void (*)(void *)) crypto_proc, NULL, 331 &cryptoproc, 0, 0, "crypto"); 332 if (error) { 333 printf("crypto_init: cannot start crypto thread; error %d", 334 error); 335 goto bad; 336 } 337 338 crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker), 339 M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 340 if (crypto_ret_workers == NULL) { 341 error = ENOMEM; 342 printf("crypto_init: cannot allocate ret workers\n"); 343 goto bad; 344 } 345 346 347 FOREACH_CRYPTO_RETW(ret_worker) { 348 TAILQ_INIT(&ret_worker->crp_ordered_ret_q); 349 TAILQ_INIT(&ret_worker->crp_ret_q); 350 TAILQ_INIT(&ret_worker->crp_ret_kq); 351 352 ret_worker->reorder_ops = 0; 353 ret_worker->reorder_cur_seq = 0; 354 355 mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF); 356 357 error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker, 358 &ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker)); 359 if (error) { 360 printf("crypto_init: cannot start cryptoret thread; error %d", 361 error); 362 goto bad; 363 } 364 } 365 366 keybuf_init(); 367 368 return 0; 369 bad: 370 crypto_destroy(); 371 return error; 372 } 373 374 /* 375 * Signal a crypto thread to terminate. We use the driver 376 * table lock to synchronize the sleep/wakeups so that we 377 * are sure the threads have terminated before we release 378 * the data structures they use. See crypto_finis below 379 * for the other half of this song-and-dance. 380 */ 381 static void 382 crypto_terminate(struct proc **pp, void *q) 383 { 384 struct proc *p; 385 386 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 387 p = *pp; 388 *pp = NULL; 389 if (p) { 390 wakeup_one(q); 391 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 392 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 393 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 394 PROC_UNLOCK(p); 395 CRYPTO_DRIVER_LOCK(); 396 } 397 } 398 399 static void 400 hmac_init_pad(const struct auth_hash *axf, const char *key, int klen, 401 void *auth_ctx, uint8_t padval) 402 { 403 uint8_t hmac_key[HMAC_MAX_BLOCK_LEN]; 404 u_int i; 405 406 KASSERT(axf->blocksize <= sizeof(hmac_key), 407 ("Invalid HMAC block size %d", axf->blocksize)); 408 409 /* 410 * If the key is larger than the block size, use the digest of 411 * the key as the key instead. 412 */ 413 memset(hmac_key, 0, sizeof(hmac_key)); 414 if (klen > axf->blocksize) { 415 axf->Init(auth_ctx); 416 axf->Update(auth_ctx, key, klen); 417 axf->Final(hmac_key, auth_ctx); 418 klen = axf->hashsize; 419 } else 420 memcpy(hmac_key, key, klen); 421 422 for (i = 0; i < axf->blocksize; i++) 423 hmac_key[i] ^= padval; 424 425 axf->Init(auth_ctx); 426 axf->Update(auth_ctx, hmac_key, axf->blocksize); 427 explicit_bzero(hmac_key, sizeof(hmac_key)); 428 } 429 430 void 431 hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen, 432 void *auth_ctx) 433 { 434 435 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL); 436 } 437 438 void 439 hmac_init_opad(const struct auth_hash *axf, const char *key, int klen, 440 void *auth_ctx) 441 { 442 443 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL); 444 } 445 446 static void 447 crypto_destroy(void) 448 { 449 struct crypto_ret_worker *ret_worker; 450 451 /* 452 * Terminate any crypto threads. 453 */ 454 if (crypto_tq != NULL) 455 taskqueue_drain_all(crypto_tq); 456 CRYPTO_DRIVER_LOCK(); 457 crypto_terminate(&cryptoproc, &crp_q); 458 FOREACH_CRYPTO_RETW(ret_worker) 459 crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q); 460 CRYPTO_DRIVER_UNLOCK(); 461 462 /* XXX flush queues??? */ 463 464 /* 465 * Reclaim dynamically allocated resources. 466 */ 467 if (crypto_drivers != NULL) 468 free(crypto_drivers, M_CRYPTO_DATA); 469 470 if (cryptoses_zone != NULL) 471 uma_zdestroy(cryptoses_zone); 472 if (cryptodesc_zone != NULL) 473 uma_zdestroy(cryptodesc_zone); 474 if (cryptop_zone != NULL) 475 uma_zdestroy(cryptop_zone); 476 mtx_destroy(&crypto_q_mtx); 477 FOREACH_CRYPTO_RETW(ret_worker) 478 mtx_destroy(&ret_worker->crypto_ret_mtx); 479 free(crypto_ret_workers, M_CRYPTO_DATA); 480 if (crypto_tq != NULL) 481 taskqueue_free(crypto_tq); 482 mtx_destroy(&crypto_drivers_mtx); 483 } 484 485 uint32_t 486 crypto_ses2hid(crypto_session_t crypto_session) 487 { 488 return (crypto_session->hid); 489 } 490 491 uint32_t 492 crypto_ses2caps(crypto_session_t crypto_session) 493 { 494 return (crypto_session->capabilities); 495 } 496 497 void * 498 crypto_get_driver_session(crypto_session_t crypto_session) 499 { 500 return (crypto_session->softc); 501 } 502 503 static struct cryptocap * 504 crypto_checkdriver(u_int32_t hid) 505 { 506 if (crypto_drivers == NULL) 507 return NULL; 508 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 509 } 510 511 /* 512 * Compare a driver's list of supported algorithms against another 513 * list; return non-zero if all algorithms are supported. 514 */ 515 static int 516 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) 517 { 518 const struct cryptoini *cr; 519 520 /* See if all the algorithms are supported. */ 521 for (cr = cri; cr; cr = cr->cri_next) 522 if (cap->cc_alg[cr->cri_alg] == 0) 523 return 0; 524 return 1; 525 } 526 527 /* 528 * Select a driver for a new session that supports the specified 529 * algorithms and, optionally, is constrained according to the flags. 530 * The algorithm we use here is pretty stupid; just use the 531 * first driver that supports all the algorithms we need. If there 532 * are multiple drivers we choose the driver with the fewest active 533 * sessions. We prefer hardware-backed drivers to software ones. 534 * 535 * XXX We need more smarts here (in real life too, but that's 536 * XXX another story altogether). 537 */ 538 static struct cryptocap * 539 crypto_select_driver(const struct cryptoini *cri, int flags) 540 { 541 struct cryptocap *cap, *best; 542 int match, hid; 543 544 CRYPTO_DRIVER_ASSERT(); 545 546 /* 547 * Look first for hardware crypto devices if permitted. 548 */ 549 if (flags & CRYPTOCAP_F_HARDWARE) 550 match = CRYPTOCAP_F_HARDWARE; 551 else 552 match = CRYPTOCAP_F_SOFTWARE; 553 best = NULL; 554 again: 555 for (hid = 0; hid < crypto_drivers_num; hid++) { 556 cap = &crypto_drivers[hid]; 557 /* 558 * If it's not initialized, is in the process of 559 * going away, or is not appropriate (hardware 560 * or software based on match), then skip. 561 */ 562 if (cap->cc_dev == NULL || 563 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 564 (cap->cc_flags & match) == 0) 565 continue; 566 567 /* verify all the algorithms are supported. */ 568 if (driver_suitable(cap, cri)) { 569 if (best == NULL || 570 cap->cc_sessions < best->cc_sessions) 571 best = cap; 572 } 573 } 574 if (best == NULL && match == CRYPTOCAP_F_HARDWARE && 575 (flags & CRYPTOCAP_F_SOFTWARE)) { 576 /* sort of an Algol 68-style for loop */ 577 match = CRYPTOCAP_F_SOFTWARE; 578 goto again; 579 } 580 return best; 581 } 582 583 /* 584 * Create a new session. The crid argument specifies a crypto 585 * driver to use or constraints on a driver to select (hardware 586 * only, software only, either). Whatever driver is selected 587 * must be capable of the requested crypto algorithms. 588 */ 589 int 590 crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid) 591 { 592 crypto_session_t res; 593 void *softc_mem; 594 struct cryptocap *cap; 595 u_int32_t hid; 596 size_t softc_size; 597 int err; 598 599 restart: 600 res = NULL; 601 softc_mem = NULL; 602 603 CRYPTO_DRIVER_LOCK(); 604 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 605 /* 606 * Use specified driver; verify it is capable. 607 */ 608 cap = crypto_checkdriver(crid); 609 if (cap != NULL && !driver_suitable(cap, cri)) 610 cap = NULL; 611 } else { 612 /* 613 * No requested driver; select based on crid flags. 614 */ 615 cap = crypto_select_driver(cri, crid); 616 /* 617 * if NULL then can't do everything in one session. 618 * XXX Fix this. We need to inject a "virtual" session 619 * XXX layer right about here. 620 */ 621 } 622 if (cap == NULL) { 623 CRYPTDEB("no driver"); 624 err = EOPNOTSUPP; 625 goto out; 626 } 627 cap->cc_sessions++; 628 softc_size = cap->cc_session_size; 629 hid = cap - crypto_drivers; 630 cap = NULL; 631 CRYPTO_DRIVER_UNLOCK(); 632 633 softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO); 634 res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO); 635 res->softc = softc_mem; 636 637 CRYPTO_DRIVER_LOCK(); 638 cap = crypto_checkdriver(hid); 639 if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) { 640 cap->cc_sessions--; 641 crypto_remove(cap); 642 cap = NULL; 643 } 644 if (cap == NULL) { 645 free(softc_mem, M_CRYPTO_DATA); 646 uma_zfree(cryptoses_zone, res); 647 CRYPTO_DRIVER_UNLOCK(); 648 goto restart; 649 } 650 651 /* Call the driver initialization routine. */ 652 err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri); 653 if (err != 0) { 654 CRYPTDEB("dev newsession failed: %d", err); 655 goto out; 656 } 657 658 res->capabilities = cap->cc_flags & 0xff000000; 659 res->hid = hid; 660 *cses = res; 661 662 out: 663 CRYPTO_DRIVER_UNLOCK(); 664 if (err != 0) { 665 free(softc_mem, M_CRYPTO_DATA); 666 if (res != NULL) 667 uma_zfree(cryptoses_zone, res); 668 } 669 return err; 670 } 671 672 static void 673 crypto_remove(struct cryptocap *cap) 674 { 675 676 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 677 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 678 bzero(cap, sizeof(*cap)); 679 } 680 681 /* 682 * Delete an existing session (or a reserved session on an unregistered 683 * driver). 684 */ 685 void 686 crypto_freesession(crypto_session_t cses) 687 { 688 struct cryptocap *cap; 689 void *ses; 690 size_t ses_size; 691 u_int32_t hid; 692 693 if (cses == NULL) 694 return; 695 696 CRYPTO_DRIVER_LOCK(); 697 698 hid = crypto_ses2hid(cses); 699 KASSERT(hid < crypto_drivers_num, 700 ("bogus crypto_session %p hid %u", cses, hid)); 701 cap = &crypto_drivers[hid]; 702 703 ses = cses->softc; 704 ses_size = cap->cc_session_size; 705 706 if (cap->cc_sessions) 707 cap->cc_sessions--; 708 709 /* Call the driver cleanup routine, if available. */ 710 CRYPTODEV_FREESESSION(cap->cc_dev, cses); 711 712 explicit_bzero(ses, ses_size); 713 free(ses, M_CRYPTO_DATA); 714 uma_zfree(cryptoses_zone, cses); 715 716 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 717 crypto_remove(cap); 718 719 CRYPTO_DRIVER_UNLOCK(); 720 } 721 722 /* 723 * Return an unused driver id. Used by drivers prior to registering 724 * support for the algorithms they handle. 725 */ 726 int32_t 727 crypto_get_driverid(device_t dev, size_t sessionsize, int flags) 728 { 729 struct cryptocap *newdrv; 730 int i; 731 732 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 733 printf("%s: no flags specified when registering driver\n", 734 device_get_nameunit(dev)); 735 return -1; 736 } 737 738 CRYPTO_DRIVER_LOCK(); 739 740 for (i = 0; i < crypto_drivers_num; i++) { 741 if (crypto_drivers[i].cc_dev == NULL && 742 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 743 break; 744 } 745 } 746 747 /* Out of entries, allocate some more. */ 748 if (i == crypto_drivers_num) { 749 /* Be careful about wrap-around. */ 750 if (2 * crypto_drivers_num <= crypto_drivers_num) { 751 CRYPTO_DRIVER_UNLOCK(); 752 printf("crypto: driver count wraparound!\n"); 753 return -1; 754 } 755 756 newdrv = malloc(2 * crypto_drivers_num * 757 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 758 if (newdrv == NULL) { 759 CRYPTO_DRIVER_UNLOCK(); 760 printf("crypto: no space to expand driver table!\n"); 761 return -1; 762 } 763 764 bcopy(crypto_drivers, newdrv, 765 crypto_drivers_num * sizeof(struct cryptocap)); 766 767 crypto_drivers_num *= 2; 768 769 free(crypto_drivers, M_CRYPTO_DATA); 770 crypto_drivers = newdrv; 771 } 772 773 /* NB: state is zero'd on free */ 774 crypto_drivers[i].cc_sessions = 1; /* Mark */ 775 crypto_drivers[i].cc_dev = dev; 776 crypto_drivers[i].cc_flags = flags; 777 crypto_drivers[i].cc_session_size = sessionsize; 778 if (bootverbose) 779 printf("crypto: assign %s driver id %u, flags 0x%x\n", 780 device_get_nameunit(dev), i, flags); 781 782 CRYPTO_DRIVER_UNLOCK(); 783 784 return i; 785 } 786 787 /* 788 * Lookup a driver by name. We match against the full device 789 * name and unit, and against just the name. The latter gives 790 * us a simple widlcarding by device name. On success return the 791 * driver/hardware identifier; otherwise return -1. 792 */ 793 int 794 crypto_find_driver(const char *match) 795 { 796 int i, len = strlen(match); 797 798 CRYPTO_DRIVER_LOCK(); 799 for (i = 0; i < crypto_drivers_num; i++) { 800 device_t dev = crypto_drivers[i].cc_dev; 801 if (dev == NULL || 802 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) 803 continue; 804 if (strncmp(match, device_get_nameunit(dev), len) == 0 || 805 strncmp(match, device_get_name(dev), len) == 0) 806 break; 807 } 808 CRYPTO_DRIVER_UNLOCK(); 809 return i < crypto_drivers_num ? i : -1; 810 } 811 812 /* 813 * Return the device_t for the specified driver or NULL 814 * if the driver identifier is invalid. 815 */ 816 device_t 817 crypto_find_device_byhid(int hid) 818 { 819 struct cryptocap *cap = crypto_checkdriver(hid); 820 return cap != NULL ? cap->cc_dev : NULL; 821 } 822 823 /* 824 * Return the device/driver capabilities. 825 */ 826 int 827 crypto_getcaps(int hid) 828 { 829 struct cryptocap *cap = crypto_checkdriver(hid); 830 return cap != NULL ? cap->cc_flags : 0; 831 } 832 833 /* 834 * Register support for a key-related algorithm. This routine 835 * is called once for each algorithm supported a driver. 836 */ 837 int 838 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) 839 { 840 struct cryptocap *cap; 841 int err; 842 843 CRYPTO_DRIVER_LOCK(); 844 845 cap = crypto_checkdriver(driverid); 846 if (cap != NULL && 847 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 848 /* 849 * XXX Do some performance testing to determine placing. 850 * XXX We probably need an auxiliary data structure that 851 * XXX describes relative performances. 852 */ 853 854 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 855 if (bootverbose) 856 printf("crypto: %s registers key alg %u flags %u\n" 857 , device_get_nameunit(cap->cc_dev) 858 , kalg 859 , flags 860 ); 861 gone_in_dev(cap->cc_dev, 14, "asymmetric crypto"); 862 err = 0; 863 } else 864 err = EINVAL; 865 866 CRYPTO_DRIVER_UNLOCK(); 867 return err; 868 } 869 870 /* 871 * Register support for a non-key-related algorithm. This routine 872 * is called once for each such algorithm supported by a driver. 873 */ 874 int 875 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 876 u_int32_t flags) 877 { 878 struct cryptocap *cap; 879 int err; 880 881 CRYPTO_DRIVER_LOCK(); 882 883 cap = crypto_checkdriver(driverid); 884 /* NB: algorithms are in the range [1..max] */ 885 if (cap != NULL && 886 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 887 /* 888 * XXX Do some performance testing to determine placing. 889 * XXX We probably need an auxiliary data structure that 890 * XXX describes relative performances. 891 */ 892 893 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 894 cap->cc_max_op_len[alg] = maxoplen; 895 if (bootverbose) 896 printf("crypto: %s registers alg %u flags %u maxoplen %u\n" 897 , device_get_nameunit(cap->cc_dev) 898 , alg 899 , flags 900 , maxoplen 901 ); 902 cap->cc_sessions = 0; /* Unmark */ 903 err = 0; 904 } else 905 err = EINVAL; 906 907 CRYPTO_DRIVER_UNLOCK(); 908 return err; 909 } 910 911 static void 912 driver_finis(struct cryptocap *cap) 913 { 914 u_int32_t ses, kops; 915 916 CRYPTO_DRIVER_ASSERT(); 917 918 ses = cap->cc_sessions; 919 kops = cap->cc_koperations; 920 bzero(cap, sizeof(*cap)); 921 if (ses != 0 || kops != 0) { 922 /* 923 * If there are pending sessions, 924 * just mark as invalid. 925 */ 926 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 927 cap->cc_sessions = ses; 928 cap->cc_koperations = kops; 929 } 930 } 931 932 /* 933 * Unregister a crypto driver. If there are pending sessions using it, 934 * leave enough information around so that subsequent calls using those 935 * sessions will correctly detect the driver has been unregistered and 936 * reroute requests. 937 */ 938 int 939 crypto_unregister(u_int32_t driverid, int alg) 940 { 941 struct cryptocap *cap; 942 int i, err; 943 944 CRYPTO_DRIVER_LOCK(); 945 cap = crypto_checkdriver(driverid); 946 if (cap != NULL && 947 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 948 cap->cc_alg[alg] != 0) { 949 cap->cc_alg[alg] = 0; 950 cap->cc_max_op_len[alg] = 0; 951 952 /* Was this the last algorithm ? */ 953 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 954 if (cap->cc_alg[i] != 0) 955 break; 956 957 if (i == CRYPTO_ALGORITHM_MAX + 1) 958 driver_finis(cap); 959 err = 0; 960 } else 961 err = EINVAL; 962 CRYPTO_DRIVER_UNLOCK(); 963 964 return err; 965 } 966 967 /* 968 * Unregister all algorithms associated with a crypto driver. 969 * If there are pending sessions using it, leave enough information 970 * around so that subsequent calls using those sessions will 971 * correctly detect the driver has been unregistered and reroute 972 * requests. 973 */ 974 int 975 crypto_unregister_all(u_int32_t driverid) 976 { 977 struct cryptocap *cap; 978 int err; 979 980 CRYPTO_DRIVER_LOCK(); 981 cap = crypto_checkdriver(driverid); 982 if (cap != NULL) { 983 driver_finis(cap); 984 err = 0; 985 } else 986 err = EINVAL; 987 CRYPTO_DRIVER_UNLOCK(); 988 989 return err; 990 } 991 992 /* 993 * Clear blockage on a driver. The what parameter indicates whether 994 * the driver is now ready for cryptop's and/or cryptokop's. 995 */ 996 int 997 crypto_unblock(u_int32_t driverid, int what) 998 { 999 struct cryptocap *cap; 1000 int err; 1001 1002 CRYPTO_Q_LOCK(); 1003 cap = crypto_checkdriver(driverid); 1004 if (cap != NULL) { 1005 if (what & CRYPTO_SYMQ) 1006 cap->cc_qblocked = 0; 1007 if (what & CRYPTO_ASYMQ) 1008 cap->cc_kqblocked = 0; 1009 if (crp_sleep) 1010 wakeup_one(&crp_q); 1011 err = 0; 1012 } else 1013 err = EINVAL; 1014 CRYPTO_Q_UNLOCK(); 1015 1016 return err; 1017 } 1018 1019 /* 1020 * Add a crypto request to a queue, to be processed by the kernel thread. 1021 */ 1022 int 1023 crypto_dispatch(struct cryptop *crp) 1024 { 1025 struct cryptocap *cap; 1026 u_int32_t hid; 1027 int result; 1028 1029 CRYPTOSTAT_INC(cs_ops); 1030 1031 crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num; 1032 1033 if (CRYPTOP_ASYNC(crp)) { 1034 if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { 1035 struct crypto_ret_worker *ret_worker; 1036 1037 ret_worker = CRYPTO_RETW(crp->crp_retw_id); 1038 1039 CRYPTO_RETW_LOCK(ret_worker); 1040 crp->crp_seq = ret_worker->reorder_ops++; 1041 CRYPTO_RETW_UNLOCK(ret_worker); 1042 } 1043 1044 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); 1045 taskqueue_enqueue(crypto_tq, &crp->crp_task); 1046 return (0); 1047 } 1048 1049 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 1050 hid = crypto_ses2hid(crp->crp_session); 1051 1052 /* 1053 * Caller marked the request to be processed 1054 * immediately; dispatch it directly to the 1055 * driver unless the driver is currently blocked. 1056 */ 1057 cap = crypto_checkdriver(hid); 1058 /* Driver cannot disappeared when there is an active session. */ 1059 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 1060 if (!cap->cc_qblocked) { 1061 result = crypto_invoke(cap, crp, 0); 1062 if (result != ERESTART) 1063 return (result); 1064 /* 1065 * The driver ran out of resources, put the request on 1066 * the queue. 1067 */ 1068 } 1069 } 1070 crypto_batch_enqueue(crp); 1071 return 0; 1072 } 1073 1074 void 1075 crypto_batch_enqueue(struct cryptop *crp) 1076 { 1077 1078 CRYPTO_Q_LOCK(); 1079 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 1080 if (crp_sleep) 1081 wakeup_one(&crp_q); 1082 CRYPTO_Q_UNLOCK(); 1083 } 1084 1085 /* 1086 * Add an asymetric crypto request to a queue, 1087 * to be processed by the kernel thread. 1088 */ 1089 int 1090 crypto_kdispatch(struct cryptkop *krp) 1091 { 1092 int error; 1093 1094 CRYPTOSTAT_INC(cs_kops); 1095 1096 error = crypto_kinvoke(krp, krp->krp_crid); 1097 if (error == ERESTART) { 1098 CRYPTO_Q_LOCK(); 1099 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 1100 if (crp_sleep) 1101 wakeup_one(&crp_q); 1102 CRYPTO_Q_UNLOCK(); 1103 error = 0; 1104 } 1105 return error; 1106 } 1107 1108 /* 1109 * Verify a driver is suitable for the specified operation. 1110 */ 1111 static __inline int 1112 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) 1113 { 1114 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; 1115 } 1116 1117 /* 1118 * Select a driver for an asym operation. The driver must 1119 * support the necessary algorithm. The caller can constrain 1120 * which device is selected with the flags parameter. The 1121 * algorithm we use here is pretty stupid; just use the first 1122 * driver that supports the algorithms we need. If there are 1123 * multiple suitable drivers we choose the driver with the 1124 * fewest active operations. We prefer hardware-backed 1125 * drivers to software ones when either may be used. 1126 */ 1127 static struct cryptocap * 1128 crypto_select_kdriver(const struct cryptkop *krp, int flags) 1129 { 1130 struct cryptocap *cap, *best; 1131 int match, hid; 1132 1133 CRYPTO_DRIVER_ASSERT(); 1134 1135 /* 1136 * Look first for hardware crypto devices if permitted. 1137 */ 1138 if (flags & CRYPTOCAP_F_HARDWARE) 1139 match = CRYPTOCAP_F_HARDWARE; 1140 else 1141 match = CRYPTOCAP_F_SOFTWARE; 1142 best = NULL; 1143 again: 1144 for (hid = 0; hid < crypto_drivers_num; hid++) { 1145 cap = &crypto_drivers[hid]; 1146 /* 1147 * If it's not initialized, is in the process of 1148 * going away, or is not appropriate (hardware 1149 * or software based on match), then skip. 1150 */ 1151 if (cap->cc_dev == NULL || 1152 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 1153 (cap->cc_flags & match) == 0) 1154 continue; 1155 1156 /* verify all the algorithms are supported. */ 1157 if (kdriver_suitable(cap, krp)) { 1158 if (best == NULL || 1159 cap->cc_koperations < best->cc_koperations) 1160 best = cap; 1161 } 1162 } 1163 if (best != NULL) 1164 return best; 1165 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 1166 /* sort of an Algol 68-style for loop */ 1167 match = CRYPTOCAP_F_SOFTWARE; 1168 goto again; 1169 } 1170 return best; 1171 } 1172 1173 /* 1174 * Dispatch an asymmetric crypto request. 1175 */ 1176 static int 1177 crypto_kinvoke(struct cryptkop *krp, int crid) 1178 { 1179 struct cryptocap *cap = NULL; 1180 int error; 1181 1182 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 1183 KASSERT(krp->krp_callback != NULL, 1184 ("%s: krp->crp_callback == NULL", __func__)); 1185 1186 CRYPTO_DRIVER_LOCK(); 1187 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 1188 cap = crypto_checkdriver(crid); 1189 if (cap != NULL) { 1190 /* 1191 * Driver present, it must support the necessary 1192 * algorithm and, if s/w drivers are excluded, 1193 * it must be registered as hardware-backed. 1194 */ 1195 if (!kdriver_suitable(cap, krp) || 1196 (!crypto_devallowsoft && 1197 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) 1198 cap = NULL; 1199 } 1200 } else { 1201 /* 1202 * No requested driver; select based on crid flags. 1203 */ 1204 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ 1205 crid &= ~CRYPTOCAP_F_SOFTWARE; 1206 cap = crypto_select_kdriver(krp, crid); 1207 } 1208 if (cap != NULL && !cap->cc_kqblocked) { 1209 krp->krp_hid = cap - crypto_drivers; 1210 cap->cc_koperations++; 1211 CRYPTO_DRIVER_UNLOCK(); 1212 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); 1213 CRYPTO_DRIVER_LOCK(); 1214 if (error == ERESTART) { 1215 cap->cc_koperations--; 1216 CRYPTO_DRIVER_UNLOCK(); 1217 return (error); 1218 } 1219 } else { 1220 /* 1221 * NB: cap is !NULL if device is blocked; in 1222 * that case return ERESTART so the operation 1223 * is resubmitted if possible. 1224 */ 1225 error = (cap == NULL) ? ENODEV : ERESTART; 1226 } 1227 CRYPTO_DRIVER_UNLOCK(); 1228 1229 if (error) { 1230 krp->krp_status = error; 1231 crypto_kdone(krp); 1232 } 1233 return 0; 1234 } 1235 1236 static void 1237 crypto_task_invoke(void *ctx, int pending) 1238 { 1239 struct cryptocap *cap; 1240 struct cryptop *crp; 1241 int hid, result; 1242 1243 crp = (struct cryptop *)ctx; 1244 1245 hid = crypto_ses2hid(crp->crp_session); 1246 cap = crypto_checkdriver(hid); 1247 1248 result = crypto_invoke(cap, crp, 0); 1249 if (result == ERESTART) 1250 crypto_batch_enqueue(crp); 1251 } 1252 1253 /* 1254 * Dispatch a crypto request to the appropriate crypto devices. 1255 */ 1256 static int 1257 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 1258 { 1259 1260 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 1261 KASSERT(crp->crp_callback != NULL, 1262 ("%s: crp->crp_callback == NULL", __func__)); 1263 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 1264 1265 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 1266 struct cryptodesc *crd; 1267 crypto_session_t nses; 1268 1269 /* 1270 * Driver has unregistered; migrate the session and return 1271 * an error to the caller so they'll resubmit the op. 1272 * 1273 * XXX: What if there are more already queued requests for this 1274 * session? 1275 */ 1276 crypto_freesession(crp->crp_session); 1277 1278 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1279 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1280 1281 /* XXX propagate flags from initial session? */ 1282 if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI), 1283 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) 1284 crp->crp_session = nses; 1285 1286 crp->crp_etype = EAGAIN; 1287 crypto_done(crp); 1288 return 0; 1289 } else { 1290 /* 1291 * Invoke the driver to process the request. 1292 */ 1293 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); 1294 } 1295 } 1296 1297 /* 1298 * Release a set of crypto descriptors. 1299 */ 1300 void 1301 crypto_freereq(struct cryptop *crp) 1302 { 1303 struct cryptodesc *crd; 1304 1305 if (crp == NULL) 1306 return; 1307 1308 #ifdef DIAGNOSTIC 1309 { 1310 struct cryptop *crp2; 1311 struct crypto_ret_worker *ret_worker; 1312 1313 CRYPTO_Q_LOCK(); 1314 TAILQ_FOREACH(crp2, &crp_q, crp_next) { 1315 KASSERT(crp2 != crp, 1316 ("Freeing cryptop from the crypto queue (%p).", 1317 crp)); 1318 } 1319 CRYPTO_Q_UNLOCK(); 1320 1321 FOREACH_CRYPTO_RETW(ret_worker) { 1322 CRYPTO_RETW_LOCK(ret_worker); 1323 TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) { 1324 KASSERT(crp2 != crp, 1325 ("Freeing cryptop from the return queue (%p).", 1326 crp)); 1327 } 1328 CRYPTO_RETW_UNLOCK(ret_worker); 1329 } 1330 } 1331 #endif 1332 1333 while ((crd = crp->crp_desc) != NULL) { 1334 crp->crp_desc = crd->crd_next; 1335 uma_zfree(cryptodesc_zone, crd); 1336 } 1337 uma_zfree(cryptop_zone, crp); 1338 } 1339 1340 /* 1341 * Acquire a set of crypto descriptors. 1342 */ 1343 struct cryptop * 1344 crypto_getreq(int num) 1345 { 1346 struct cryptodesc *crd; 1347 struct cryptop *crp; 1348 1349 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 1350 if (crp != NULL) { 1351 while (num--) { 1352 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 1353 if (crd == NULL) { 1354 crypto_freereq(crp); 1355 return NULL; 1356 } 1357 1358 crd->crd_next = crp->crp_desc; 1359 crp->crp_desc = crd; 1360 } 1361 } 1362 return crp; 1363 } 1364 1365 /* 1366 * Invoke the callback on behalf of the driver. 1367 */ 1368 void 1369 crypto_done(struct cryptop *crp) 1370 { 1371 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 1372 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 1373 crp->crp_flags |= CRYPTO_F_DONE; 1374 if (crp->crp_etype != 0) 1375 CRYPTOSTAT_INC(cs_errs); 1376 1377 /* 1378 * CBIMM means unconditionally do the callback immediately; 1379 * CBIFSYNC means do the callback immediately only if the 1380 * operation was done synchronously. Both are used to avoid 1381 * doing extraneous context switches; the latter is mostly 1382 * used with the software crypto driver. 1383 */ 1384 if (!CRYPTOP_ASYNC_KEEPORDER(crp) && 1385 ((crp->crp_flags & CRYPTO_F_CBIMM) || 1386 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 1387 (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { 1388 /* 1389 * Do the callback directly. This is ok when the 1390 * callback routine does very little (e.g. the 1391 * /dev/crypto callback method just does a wakeup). 1392 */ 1393 crp->crp_callback(crp); 1394 } else { 1395 struct crypto_ret_worker *ret_worker; 1396 bool wake; 1397 1398 ret_worker = CRYPTO_RETW(crp->crp_retw_id); 1399 wake = false; 1400 1401 /* 1402 * Normal case; queue the callback for the thread. 1403 */ 1404 CRYPTO_RETW_LOCK(ret_worker); 1405 if (CRYPTOP_ASYNC_KEEPORDER(crp)) { 1406 struct cryptop *tmp; 1407 1408 TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q, 1409 cryptop_q, crp_next) { 1410 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { 1411 TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q, 1412 tmp, crp, crp_next); 1413 break; 1414 } 1415 } 1416 if (tmp == NULL) { 1417 TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q, 1418 crp, crp_next); 1419 } 1420 1421 if (crp->crp_seq == ret_worker->reorder_cur_seq) 1422 wake = true; 1423 } 1424 else { 1425 if (CRYPTO_RETW_EMPTY(ret_worker)) 1426 wake = true; 1427 1428 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); 1429 } 1430 1431 if (wake) 1432 wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ 1433 CRYPTO_RETW_UNLOCK(ret_worker); 1434 } 1435 } 1436 1437 /* 1438 * Invoke the callback on behalf of the driver. 1439 */ 1440 void 1441 crypto_kdone(struct cryptkop *krp) 1442 { 1443 struct crypto_ret_worker *ret_worker; 1444 struct cryptocap *cap; 1445 1446 if (krp->krp_status != 0) 1447 CRYPTOSTAT_INC(cs_kerrs); 1448 CRYPTO_DRIVER_LOCK(); 1449 /* XXX: What if driver is loaded in the meantime? */ 1450 if (krp->krp_hid < crypto_drivers_num) { 1451 cap = &crypto_drivers[krp->krp_hid]; 1452 KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); 1453 cap->cc_koperations--; 1454 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1455 crypto_remove(cap); 1456 } 1457 CRYPTO_DRIVER_UNLOCK(); 1458 1459 ret_worker = CRYPTO_RETW(0); 1460 1461 CRYPTO_RETW_LOCK(ret_worker); 1462 if (CRYPTO_RETW_EMPTY(ret_worker)) 1463 wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ 1464 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); 1465 CRYPTO_RETW_UNLOCK(ret_worker); 1466 } 1467 1468 int 1469 crypto_getfeat(int *featp) 1470 { 1471 int hid, kalg, feat = 0; 1472 1473 CRYPTO_DRIVER_LOCK(); 1474 for (hid = 0; hid < crypto_drivers_num; hid++) { 1475 const struct cryptocap *cap = &crypto_drivers[hid]; 1476 1477 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1478 !crypto_devallowsoft) { 1479 continue; 1480 } 1481 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1482 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) 1483 feat |= 1 << kalg; 1484 } 1485 CRYPTO_DRIVER_UNLOCK(); 1486 *featp = feat; 1487 return (0); 1488 } 1489 1490 /* 1491 * Terminate a thread at module unload. The process that 1492 * initiated this is waiting for us to signal that we're gone; 1493 * wake it up and exit. We use the driver table lock to insure 1494 * we don't do the wakeup before they're waiting. There is no 1495 * race here because the waiter sleeps on the proc lock for the 1496 * thread so it gets notified at the right time because of an 1497 * extra wakeup that's done in exit1(). 1498 */ 1499 static void 1500 crypto_finis(void *chan) 1501 { 1502 CRYPTO_DRIVER_LOCK(); 1503 wakeup_one(chan); 1504 CRYPTO_DRIVER_UNLOCK(); 1505 kproc_exit(0); 1506 } 1507 1508 /* 1509 * Crypto thread, dispatches crypto requests. 1510 */ 1511 static void 1512 crypto_proc(void) 1513 { 1514 struct cryptop *crp, *submit; 1515 struct cryptkop *krp; 1516 struct cryptocap *cap; 1517 u_int32_t hid; 1518 int result, hint; 1519 1520 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1521 fpu_kern_thread(FPU_KERN_NORMAL); 1522 #endif 1523 1524 CRYPTO_Q_LOCK(); 1525 for (;;) { 1526 /* 1527 * Find the first element in the queue that can be 1528 * processed and look-ahead to see if multiple ops 1529 * are ready for the same driver. 1530 */ 1531 submit = NULL; 1532 hint = 0; 1533 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1534 hid = crypto_ses2hid(crp->crp_session); 1535 cap = crypto_checkdriver(hid); 1536 /* 1537 * Driver cannot disappeared when there is an active 1538 * session. 1539 */ 1540 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1541 __func__, __LINE__)); 1542 if (cap == NULL || cap->cc_dev == NULL) { 1543 /* Op needs to be migrated, process it. */ 1544 if (submit == NULL) 1545 submit = crp; 1546 break; 1547 } 1548 if (!cap->cc_qblocked) { 1549 if (submit != NULL) { 1550 /* 1551 * We stop on finding another op, 1552 * regardless whether its for the same 1553 * driver or not. We could keep 1554 * searching the queue but it might be 1555 * better to just use a per-driver 1556 * queue instead. 1557 */ 1558 if (crypto_ses2hid(submit->crp_session) == hid) 1559 hint = CRYPTO_HINT_MORE; 1560 break; 1561 } else { 1562 submit = crp; 1563 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1564 break; 1565 /* keep scanning for more are q'd */ 1566 } 1567 } 1568 } 1569 if (submit != NULL) { 1570 TAILQ_REMOVE(&crp_q, submit, crp_next); 1571 hid = crypto_ses2hid(submit->crp_session); 1572 cap = crypto_checkdriver(hid); 1573 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1574 __func__, __LINE__)); 1575 result = crypto_invoke(cap, submit, hint); 1576 if (result == ERESTART) { 1577 /* 1578 * The driver ran out of resources, mark the 1579 * driver ``blocked'' for cryptop's and put 1580 * the request back in the queue. It would 1581 * best to put the request back where we got 1582 * it but that's hard so for now we put it 1583 * at the front. This should be ok; putting 1584 * it at the end does not work. 1585 */ 1586 /* XXX validate sid again? */ 1587 crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1; 1588 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1589 CRYPTOSTAT_INC(cs_blocks); 1590 } 1591 } 1592 1593 /* As above, but for key ops */ 1594 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1595 cap = crypto_checkdriver(krp->krp_hid); 1596 if (cap == NULL || cap->cc_dev == NULL) { 1597 /* 1598 * Operation needs to be migrated, invalidate 1599 * the assigned device so it will reselect a 1600 * new one below. Propagate the original 1601 * crid selection flags if supplied. 1602 */ 1603 krp->krp_hid = krp->krp_crid & 1604 (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); 1605 if (krp->krp_hid == 0) 1606 krp->krp_hid = 1607 CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; 1608 break; 1609 } 1610 if (!cap->cc_kqblocked) 1611 break; 1612 } 1613 if (krp != NULL) { 1614 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1615 result = crypto_kinvoke(krp, krp->krp_hid); 1616 if (result == ERESTART) { 1617 /* 1618 * The driver ran out of resources, mark the 1619 * driver ``blocked'' for cryptkop's and put 1620 * the request back in the queue. It would 1621 * best to put the request back where we got 1622 * it but that's hard so for now we put it 1623 * at the front. This should be ok; putting 1624 * it at the end does not work. 1625 */ 1626 /* XXX validate sid again? */ 1627 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1628 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1629 CRYPTOSTAT_INC(cs_kblocks); 1630 } 1631 } 1632 1633 if (submit == NULL && krp == NULL) { 1634 /* 1635 * Nothing more to be processed. Sleep until we're 1636 * woken because there are more ops to process. 1637 * This happens either by submission or by a driver 1638 * becoming unblocked and notifying us through 1639 * crypto_unblock. Note that when we wakeup we 1640 * start processing each queue again from the 1641 * front. It's not clear that it's important to 1642 * preserve this ordering since ops may finish 1643 * out of order if dispatched to different devices 1644 * and some become blocked while others do not. 1645 */ 1646 crp_sleep = 1; 1647 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1648 crp_sleep = 0; 1649 if (cryptoproc == NULL) 1650 break; 1651 CRYPTOSTAT_INC(cs_intrs); 1652 } 1653 } 1654 CRYPTO_Q_UNLOCK(); 1655 1656 crypto_finis(&crp_q); 1657 } 1658 1659 /* 1660 * Crypto returns thread, does callbacks for processed crypto requests. 1661 * Callbacks are done here, rather than in the crypto drivers, because 1662 * callbacks typically are expensive and would slow interrupt handling. 1663 */ 1664 static void 1665 crypto_ret_proc(struct crypto_ret_worker *ret_worker) 1666 { 1667 struct cryptop *crpt; 1668 struct cryptkop *krpt; 1669 1670 CRYPTO_RETW_LOCK(ret_worker); 1671 for (;;) { 1672 /* Harvest return q's for completed ops */ 1673 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q); 1674 if (crpt != NULL) { 1675 if (crpt->crp_seq == ret_worker->reorder_cur_seq) { 1676 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next); 1677 ret_worker->reorder_cur_seq++; 1678 } else { 1679 crpt = NULL; 1680 } 1681 } 1682 1683 if (crpt == NULL) { 1684 crpt = TAILQ_FIRST(&ret_worker->crp_ret_q); 1685 if (crpt != NULL) 1686 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next); 1687 } 1688 1689 krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq); 1690 if (krpt != NULL) 1691 TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next); 1692 1693 if (crpt != NULL || krpt != NULL) { 1694 CRYPTO_RETW_UNLOCK(ret_worker); 1695 /* 1696 * Run callbacks unlocked. 1697 */ 1698 if (crpt != NULL) 1699 crpt->crp_callback(crpt); 1700 if (krpt != NULL) 1701 krpt->krp_callback(krpt); 1702 CRYPTO_RETW_LOCK(ret_worker); 1703 } else { 1704 /* 1705 * Nothing more to be processed. Sleep until we're 1706 * woken because there are more returns to process. 1707 */ 1708 msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT, 1709 "crypto_ret_wait", 0); 1710 if (ret_worker->cryptoretproc == NULL) 1711 break; 1712 CRYPTOSTAT_INC(cs_rets); 1713 } 1714 } 1715 CRYPTO_RETW_UNLOCK(ret_worker); 1716 1717 crypto_finis(&ret_worker->crp_ret_q); 1718 } 1719 1720 #ifdef DDB 1721 static void 1722 db_show_drivers(void) 1723 { 1724 int hid; 1725 1726 db_printf("%12s %4s %4s %8s %2s %2s\n" 1727 , "Device" 1728 , "Ses" 1729 , "Kops" 1730 , "Flags" 1731 , "QB" 1732 , "KB" 1733 ); 1734 for (hid = 0; hid < crypto_drivers_num; hid++) { 1735 const struct cryptocap *cap = &crypto_drivers[hid]; 1736 if (cap->cc_dev == NULL) 1737 continue; 1738 db_printf("%-12s %4u %4u %08x %2u %2u\n" 1739 , device_get_nameunit(cap->cc_dev) 1740 , cap->cc_sessions 1741 , cap->cc_koperations 1742 , cap->cc_flags 1743 , cap->cc_qblocked 1744 , cap->cc_kqblocked 1745 ); 1746 } 1747 } 1748 1749 DB_SHOW_COMMAND(crypto, db_show_crypto) 1750 { 1751 struct cryptop *crp; 1752 struct crypto_ret_worker *ret_worker; 1753 1754 db_show_drivers(); 1755 db_printf("\n"); 1756 1757 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", 1758 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", 1759 "Desc", "Callback"); 1760 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1761 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" 1762 , (int) crypto_ses2hid(crp->crp_session) 1763 , (int) crypto_ses2caps(crp->crp_session) 1764 , crp->crp_ilen, crp->crp_olen 1765 , crp->crp_etype 1766 , crp->crp_flags 1767 , crp->crp_desc 1768 , crp->crp_callback 1769 ); 1770 } 1771 FOREACH_CRYPTO_RETW(ret_worker) { 1772 db_printf("\n%8s %4s %4s %4s %8s\n", 1773 "ret_worker", "HID", "Etype", "Flags", "Callback"); 1774 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { 1775 TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) { 1776 db_printf("%8td %4u %4u %04x %8p\n" 1777 , CRYPTO_RETW_ID(ret_worker) 1778 , (int) crypto_ses2hid(crp->crp_session) 1779 , crp->crp_etype 1780 , crp->crp_flags 1781 , crp->crp_callback 1782 ); 1783 } 1784 } 1785 } 1786 } 1787 1788 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) 1789 { 1790 struct cryptkop *krp; 1791 struct crypto_ret_worker *ret_worker; 1792 1793 db_show_drivers(); 1794 db_printf("\n"); 1795 1796 db_printf("%4s %5s %4s %4s %8s %4s %8s\n", 1797 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); 1798 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1799 db_printf("%4u %5u %4u %4u %08x %4u %8p\n" 1800 , krp->krp_op 1801 , krp->krp_status 1802 , krp->krp_iparams, krp->krp_oparams 1803 , krp->krp_crid, krp->krp_hid 1804 , krp->krp_callback 1805 ); 1806 } 1807 1808 ret_worker = CRYPTO_RETW(0); 1809 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { 1810 db_printf("%4s %5s %8s %4s %8s\n", 1811 "Op", "Status", "CRID", "HID", "Callback"); 1812 TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) { 1813 db_printf("%4u %5u %08x %4u %8p\n" 1814 , krp->krp_op 1815 , krp->krp_status 1816 , krp->krp_crid, krp->krp_hid 1817 , krp->krp_callback 1818 ); 1819 } 1820 } 1821 } 1822 #endif 1823 1824 int crypto_modevent(module_t mod, int type, void *unused); 1825 1826 /* 1827 * Initialization code, both for static and dynamic loading. 1828 * Note this is not invoked with the usual MODULE_DECLARE 1829 * mechanism but instead is listed as a dependency by the 1830 * cryptosoft driver. This guarantees proper ordering of 1831 * calls on module load/unload. 1832 */ 1833 int 1834 crypto_modevent(module_t mod, int type, void *unused) 1835 { 1836 int error = EINVAL; 1837 1838 switch (type) { 1839 case MOD_LOAD: 1840 error = crypto_init(); 1841 if (error == 0 && bootverbose) 1842 printf("crypto: <crypto core>\n"); 1843 break; 1844 case MOD_UNLOAD: 1845 /*XXX disallow if active sessions */ 1846 error = 0; 1847 crypto_destroy(); 1848 return 0; 1849 } 1850 return error; 1851 } 1852 MODULE_VERSION(crypto, 1); 1853 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
Cache object: 0337050e233c8fd62941703b207676ae
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]