Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 */ 24 25 #include <sys/cdefs.h> 26 __FBSDID("$FreeBSD$"); 27 28 /* 29 * Cryptographic Subsystem. 30 * 31 * This code is derived from the Openbsd Cryptographic Framework (OCF) 32 * that has the copyright shown below. Very little of the original 33 * code remains. 34 */ 35 36 /*- 37 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 38 * 39 * This code was written by Angelos D. Keromytis in Athens, Greece, in 40 * February 2000. Network Security Technologies Inc. (NSTI) kindly 41 * supported the development of this code. 42 * 43 * Copyright (c) 2000, 2001 Angelos D. Keromytis 44 * 45 * Permission to use, copy, and modify this software with or without fee 46 * is hereby granted, provided that this entire notice is included in 47 * all source code copies of any software which is or includes a copy or 48 * modification of this software. 49 * 50 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 51 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 52 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 53 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 54 * PURPOSE. 55 */ 56 57 #define CRYPTO_TIMING /* enable timing support */ 58 59 #include "opt_ddb.h" 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/eventhandler.h> 64 #include <sys/kernel.h> 65 #include <sys/kthread.h> 66 #include <sys/lock.h> 67 #include <sys/module.h> 68 #include <sys/mutex.h> 69 #include <sys/malloc.h> 70 #include <sys/proc.h> 71 #include <sys/sysctl.h> 72 73 #include <ddb/ddb.h> 74 75 #include <vm/uma.h> 76 #include <opencrypto/cryptodev.h> 77 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 78 79 #include <sys/kobj.h> 80 #include <sys/bus.h> 81 #include "cryptodev_if.h" 82 83 /* 84 * Crypto drivers register themselves by allocating a slot in the 85 * crypto_drivers table with crypto_get_driverid() and then registering 86 * each algorithm they support with crypto_register() and crypto_kregister(). 87 */ 88 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 89 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 90 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 91 #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED) 92 93 /* 94 * Crypto device/driver capabilities structure. 95 * 96 * Synchronization: 97 * (d) - protected by CRYPTO_DRIVER_LOCK() 98 * (q) - protected by CRYPTO_Q_LOCK() 99 * Not tagged fields are read-only. 100 */ 101 struct cryptocap { 102 device_t cc_dev; /* (d) device/driver */ 103 u_int32_t cc_sessions; /* (d) # of sessions */ 104 u_int32_t cc_koperations; /* (d) # os asym operations */ 105 /* 106 * Largest possible operator length (in bits) for each type of 107 * encryption algorithm. XXX not used 108 */ 109 u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; 110 u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; 111 u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; 112 113 int cc_flags; /* (d) flags */ 114 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ 115 int cc_qblocked; /* (q) symmetric q blocked */ 116 int cc_kqblocked; /* (q) asymmetric q blocked */ 117 }; 118 static struct cryptocap *crypto_drivers = NULL; 119 static int crypto_drivers_num = 0; 120 121 /* 122 * There are two queues for crypto requests; one for symmetric (e.g. 123 * cipher) operations and one for asymmetric (e.g. MOD)operations. 124 * A single mutex is used to lock access to both queues. We could 125 * have one per-queue but having one simplifies handling of block/unblock 126 * operations. 127 */ 128 static int crp_sleep = 0; 129 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 130 static TAILQ_HEAD(,cryptkop) crp_kq; 131 static struct mtx crypto_q_mtx; 132 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 133 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 134 135 /* 136 * There are two queues for processing completed crypto requests; one 137 * for the symmetric and one for the asymmetric ops. We only need one 138 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 139 * mutex is used to lock access to both queues. Note that this lock 140 * must be separate from the lock on request queues to insure driver 141 * callbacks don't generate lock order reversals. 142 */ 143 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 144 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 145 static struct mtx crypto_ret_q_mtx; 146 #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 147 #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 148 #define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) 149 150 static uma_zone_t cryptop_zone; 151 static uma_zone_t cryptodesc_zone; 152 153 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 154 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 155 &crypto_userasymcrypto, 0, 156 "Enable/disable user-mode access to asymmetric crypto support"); 157 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 158 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 159 &crypto_devallowsoft, 0, 160 "Enable/disable use of software asym crypto support"); 161 162 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 163 164 static void crypto_proc(void); 165 static struct proc *cryptoproc; 166 static void crypto_ret_proc(void); 167 static struct proc *cryptoretproc; 168 static void crypto_destroy(void); 169 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 170 static int crypto_kinvoke(struct cryptkop *krp, int flags); 171 172 static struct cryptostats cryptostats; 173 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 174 cryptostats, "Crypto system statistics"); 175 176 #ifdef CRYPTO_TIMING 177 static int crypto_timing = 0; 178 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 179 &crypto_timing, 0, "Enable/disable crypto timing support"); 180 #endif 181 182 static int 183 crypto_init(void) 184 { 185 int error; 186 187 mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 188 MTX_DEF|MTX_QUIET); 189 190 TAILQ_INIT(&crp_q); 191 TAILQ_INIT(&crp_kq); 192 mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 193 194 TAILQ_INIT(&crp_ret_q); 195 TAILQ_INIT(&crp_ret_kq); 196 mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 197 198 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 199 0, 0, 0, 0, 200 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 201 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 202 0, 0, 0, 0, 203 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 204 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 205 printf("crypto_init: cannot setup crypto zones\n"); 206 error = ENOMEM; 207 goto bad; 208 } 209 210 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 211 crypto_drivers = malloc(crypto_drivers_num * 212 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 213 if (crypto_drivers == NULL) { 214 printf("crypto_init: cannot setup crypto drivers\n"); 215 error = ENOMEM; 216 goto bad; 217 } 218 219 error = kthread_create((void (*)(void *)) crypto_proc, NULL, 220 &cryptoproc, 0, 0, "crypto"); 221 if (error) { 222 printf("crypto_init: cannot start crypto thread; error %d", 223 error); 224 goto bad; 225 } 226 227 error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 228 &cryptoretproc, 0, 0, "crypto returns"); 229 if (error) { 230 printf("crypto_init: cannot start cryptoret thread; error %d", 231 error); 232 goto bad; 233 } 234 return 0; 235 bad: 236 crypto_destroy(); 237 return error; 238 } 239 240 /* 241 * Signal a crypto thread to terminate. We use the driver 242 * table lock to synchronize the sleep/wakeups so that we 243 * are sure the threads have terminated before we release 244 * the data structures they use. See crypto_finis below 245 * for the other half of this song-and-dance. 246 */ 247 static void 248 crypto_terminate(struct proc **pp, void *q) 249 { 250 struct proc *p; 251 252 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 253 p = *pp; 254 *pp = NULL; 255 if (p) { 256 wakeup_one(q); 257 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 258 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 259 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 260 PROC_UNLOCK(p); 261 CRYPTO_DRIVER_LOCK(); 262 } 263 } 264 265 static void 266 crypto_destroy(void) 267 { 268 /* 269 * Terminate any crypto threads. 270 */ 271 CRYPTO_DRIVER_LOCK(); 272 crypto_terminate(&cryptoproc, &crp_q); 273 crypto_terminate(&cryptoretproc, &crp_ret_q); 274 CRYPTO_DRIVER_UNLOCK(); 275 276 /* XXX flush queues??? */ 277 278 /* 279 * Reclaim dynamically allocated resources. 280 */ 281 if (crypto_drivers != NULL) 282 free(crypto_drivers, M_CRYPTO_DATA); 283 284 if (cryptodesc_zone != NULL) 285 uma_zdestroy(cryptodesc_zone); 286 if (cryptop_zone != NULL) 287 uma_zdestroy(cryptop_zone); 288 mtx_destroy(&crypto_q_mtx); 289 mtx_destroy(&crypto_ret_q_mtx); 290 mtx_destroy(&crypto_drivers_mtx); 291 } 292 293 static struct cryptocap * 294 crypto_checkdriver(u_int32_t hid) 295 { 296 if (crypto_drivers == NULL) 297 return NULL; 298 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 299 } 300 301 /* 302 * Compare a driver's list of supported algorithms against another 303 * list; return non-zero if all algorithms are supported. 304 */ 305 static int 306 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) 307 { 308 const struct cryptoini *cr; 309 310 /* See if all the algorithms are supported. */ 311 for (cr = cri; cr; cr = cr->cri_next) 312 if (cap->cc_alg[cr->cri_alg] == 0) 313 return 0; 314 return 1; 315 } 316 317 /* 318 * Select a driver for a new session that supports the specified 319 * algorithms and, optionally, is constrained according to the flags. 320 * The algorithm we use here is pretty stupid; just use the 321 * first driver that supports all the algorithms we need. If there 322 * are multiple drivers we choose the driver with the fewest active 323 * sessions. We prefer hardware-backed drivers to software ones. 324 * 325 * XXX We need more smarts here (in real life too, but that's 326 * XXX another story altogether). 327 */ 328 static struct cryptocap * 329 crypto_select_driver(const struct cryptoini *cri, int flags) 330 { 331 struct cryptocap *cap, *best; 332 int match, hid; 333 334 CRYPTO_DRIVER_ASSERT(); 335 336 /* 337 * Look first for hardware crypto devices if permitted. 338 */ 339 if (flags & CRYPTOCAP_F_HARDWARE) 340 match = CRYPTOCAP_F_HARDWARE; 341 else 342 match = CRYPTOCAP_F_SOFTWARE; 343 best = NULL; 344 again: 345 for (hid = 0; hid < crypto_drivers_num; hid++) { 346 cap = &crypto_drivers[hid]; 347 /* 348 * If it's not initialized, is in the process of 349 * going away, or is not appropriate (hardware 350 * or software based on match), then skip. 351 */ 352 if (cap->cc_dev == NULL || 353 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 354 (cap->cc_flags & match) == 0) 355 continue; 356 357 /* verify all the algorithms are supported. */ 358 if (driver_suitable(cap, cri)) { 359 if (best == NULL || 360 cap->cc_sessions < best->cc_sessions) 361 best = cap; 362 } 363 } 364 if (best != NULL) 365 return best; 366 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 367 /* sort of an Algol 68-style for loop */ 368 match = CRYPTOCAP_F_SOFTWARE; 369 goto again; 370 } 371 return best; 372 } 373 374 /* 375 * Create a new session. The crid argument specifies a crypto 376 * driver to use or constraints on a driver to select (hardware 377 * only, software only, either). Whatever driver is selected 378 * must be capable of the requested crypto algorithms. 379 */ 380 int 381 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) 382 { 383 struct cryptocap *cap; 384 u_int32_t hid, lid; 385 int err; 386 387 CRYPTO_DRIVER_LOCK(); 388 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 389 /* 390 * Use specified driver; verify it is capable. 391 */ 392 cap = crypto_checkdriver(crid); 393 if (cap != NULL && !driver_suitable(cap, cri)) 394 cap = NULL; 395 } else { 396 /* 397 * No requested driver; select based on crid flags. 398 */ 399 cap = crypto_select_driver(cri, crid); 400 /* 401 * if NULL then can't do everything in one session. 402 * XXX Fix this. We need to inject a "virtual" session 403 * XXX layer right about here. 404 */ 405 } 406 if (cap != NULL) { 407 /* Call the driver initialization routine. */ 408 hid = cap - crypto_drivers; 409 lid = hid; /* Pass the driver ID. */ 410 err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri); 411 if (err == 0) { 412 (*sid) = (cap->cc_flags & 0xff000000) 413 | (hid & 0x00ffffff); 414 (*sid) <<= 32; 415 (*sid) |= (lid & 0xffffffff); 416 cap->cc_sessions++; 417 } 418 } else 419 err = EINVAL; 420 CRYPTO_DRIVER_UNLOCK(); 421 return err; 422 } 423 424 static void 425 crypto_remove(struct cryptocap *cap) 426 { 427 428 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 429 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 430 bzero(cap, sizeof(*cap)); 431 } 432 433 /* 434 * Delete an existing session (or a reserved session on an unregistered 435 * driver). 436 */ 437 int 438 crypto_freesession(u_int64_t sid) 439 { 440 struct cryptocap *cap; 441 u_int32_t hid; 442 int err; 443 444 CRYPTO_DRIVER_LOCK(); 445 446 if (crypto_drivers == NULL) { 447 err = EINVAL; 448 goto done; 449 } 450 451 /* Determine two IDs. */ 452 hid = CRYPTO_SESID2HID(sid); 453 454 if (hid >= crypto_drivers_num) { 455 err = ENOENT; 456 goto done; 457 } 458 cap = &crypto_drivers[hid]; 459 460 if (cap->cc_sessions) 461 cap->cc_sessions--; 462 463 /* Call the driver cleanup routine, if available. */ 464 err = CRYPTODEV_FREESESSION(cap->cc_dev, sid); 465 466 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 467 crypto_remove(cap); 468 469 done: 470 CRYPTO_DRIVER_UNLOCK(); 471 return err; 472 } 473 474 /* 475 * Return an unused driver id. Used by drivers prior to registering 476 * support for the algorithms they handle. 477 */ 478 int32_t 479 crypto_get_driverid(device_t dev, int flags) 480 { 481 struct cryptocap *newdrv; 482 int i; 483 484 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 485 printf("%s: no flags specified when registering driver\n", 486 device_get_nameunit(dev)); 487 return -1; 488 } 489 490 CRYPTO_DRIVER_LOCK(); 491 492 for (i = 0; i < crypto_drivers_num; i++) { 493 if (crypto_drivers[i].cc_dev == NULL && 494 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 495 break; 496 } 497 } 498 499 /* Out of entries, allocate some more. */ 500 if (i == crypto_drivers_num) { 501 /* Be careful about wrap-around. */ 502 if (2 * crypto_drivers_num <= crypto_drivers_num) { 503 CRYPTO_DRIVER_UNLOCK(); 504 printf("crypto: driver count wraparound!\n"); 505 return -1; 506 } 507 508 newdrv = malloc(2 * crypto_drivers_num * 509 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 510 if (newdrv == NULL) { 511 CRYPTO_DRIVER_UNLOCK(); 512 printf("crypto: no space to expand driver table!\n"); 513 return -1; 514 } 515 516 bcopy(crypto_drivers, newdrv, 517 crypto_drivers_num * sizeof(struct cryptocap)); 518 519 crypto_drivers_num *= 2; 520 521 free(crypto_drivers, M_CRYPTO_DATA); 522 crypto_drivers = newdrv; 523 } 524 525 /* NB: state is zero'd on free */ 526 crypto_drivers[i].cc_sessions = 1; /* Mark */ 527 crypto_drivers[i].cc_dev = dev; 528 crypto_drivers[i].cc_flags = flags; 529 if (bootverbose) 530 printf("crypto: assign %s driver id %u, flags %u\n", 531 device_get_nameunit(dev), i, flags); 532 533 CRYPTO_DRIVER_UNLOCK(); 534 535 return i; 536 } 537 538 /* 539 * Lookup a driver by name. We match against the full device 540 * name and unit, and against just the name. The latter gives 541 * us a simple widlcarding by device name. On success return the 542 * driver/hardware identifier; otherwise return -1. 543 */ 544 int 545 crypto_find_driver(const char *match) 546 { 547 int i, len = strlen(match); 548 549 CRYPTO_DRIVER_LOCK(); 550 for (i = 0; i < crypto_drivers_num; i++) { 551 device_t dev = crypto_drivers[i].cc_dev; 552 if (dev == NULL || 553 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) 554 continue; 555 if (strncmp(match, device_get_nameunit(dev), len) == 0 || 556 strncmp(match, device_get_name(dev), len) == 0) 557 break; 558 } 559 CRYPTO_DRIVER_UNLOCK(); 560 return i < crypto_drivers_num ? i : -1; 561 } 562 563 /* 564 * Return the device_t for the specified driver or NULL 565 * if the driver identifier is invalid. 566 */ 567 device_t 568 crypto_find_device_byhid(int hid) 569 { 570 struct cryptocap *cap = crypto_checkdriver(hid); 571 return cap != NULL ? cap->cc_dev : NULL; 572 } 573 574 /* 575 * Return the device/driver capabilities. 576 */ 577 int 578 crypto_getcaps(int hid) 579 { 580 struct cryptocap *cap = crypto_checkdriver(hid); 581 return cap != NULL ? cap->cc_flags : 0; 582 } 583 584 /* 585 * Register support for a key-related algorithm. This routine 586 * is called once for each algorithm supported a driver. 587 */ 588 int 589 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) 590 { 591 struct cryptocap *cap; 592 int err; 593 594 CRYPTO_DRIVER_LOCK(); 595 596 cap = crypto_checkdriver(driverid); 597 if (cap != NULL && 598 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 599 /* 600 * XXX Do some performance testing to determine placing. 601 * XXX We probably need an auxiliary data structure that 602 * XXX describes relative performances. 603 */ 604 605 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 606 if (bootverbose) 607 printf("crypto: %s registers key alg %u flags %u\n" 608 , device_get_nameunit(cap->cc_dev) 609 , kalg 610 , flags 611 ); 612 err = 0; 613 } else 614 err = EINVAL; 615 616 CRYPTO_DRIVER_UNLOCK(); 617 return err; 618 } 619 620 /* 621 * Register support for a non-key-related algorithm. This routine 622 * is called once for each such algorithm supported by a driver. 623 */ 624 int 625 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 626 u_int32_t flags) 627 { 628 struct cryptocap *cap; 629 int err; 630 631 CRYPTO_DRIVER_LOCK(); 632 633 cap = crypto_checkdriver(driverid); 634 /* NB: algorithms are in the range [1..max] */ 635 if (cap != NULL && 636 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 637 /* 638 * XXX Do some performance testing to determine placing. 639 * XXX We probably need an auxiliary data structure that 640 * XXX describes relative performances. 641 */ 642 643 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 644 cap->cc_max_op_len[alg] = maxoplen; 645 if (bootverbose) 646 printf("crypto: %s registers alg %u flags %u maxoplen %u\n" 647 , device_get_nameunit(cap->cc_dev) 648 , alg 649 , flags 650 , maxoplen 651 ); 652 cap->cc_sessions = 0; /* Unmark */ 653 err = 0; 654 } else 655 err = EINVAL; 656 657 CRYPTO_DRIVER_UNLOCK(); 658 return err; 659 } 660 661 static void 662 driver_finis(struct cryptocap *cap) 663 { 664 u_int32_t ses, kops; 665 666 CRYPTO_DRIVER_ASSERT(); 667 668 ses = cap->cc_sessions; 669 kops = cap->cc_koperations; 670 bzero(cap, sizeof(*cap)); 671 if (ses != 0 || kops != 0) { 672 /* 673 * If there are pending sessions, 674 * just mark as invalid. 675 */ 676 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 677 cap->cc_sessions = ses; 678 cap->cc_koperations = kops; 679 } 680 } 681 682 /* 683 * Unregister a crypto driver. If there are pending sessions using it, 684 * leave enough information around so that subsequent calls using those 685 * sessions will correctly detect the driver has been unregistered and 686 * reroute requests. 687 */ 688 int 689 crypto_unregister(u_int32_t driverid, int alg) 690 { 691 struct cryptocap *cap; 692 int i, err; 693 694 CRYPTO_DRIVER_LOCK(); 695 cap = crypto_checkdriver(driverid); 696 if (cap != NULL && 697 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 698 cap->cc_alg[alg] != 0) { 699 cap->cc_alg[alg] = 0; 700 cap->cc_max_op_len[alg] = 0; 701 702 /* Was this the last algorithm ? */ 703 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 704 if (cap->cc_alg[i] != 0) 705 break; 706 707 if (i == CRYPTO_ALGORITHM_MAX + 1) 708 driver_finis(cap); 709 err = 0; 710 } else 711 err = EINVAL; 712 CRYPTO_DRIVER_UNLOCK(); 713 714 return err; 715 } 716 717 /* 718 * Unregister all algorithms associated with a crypto driver. 719 * If there are pending sessions using it, leave enough information 720 * around so that subsequent calls using those sessions will 721 * correctly detect the driver has been unregistered and reroute 722 * requests. 723 */ 724 int 725 crypto_unregister_all(u_int32_t driverid) 726 { 727 struct cryptocap *cap; 728 int err; 729 730 CRYPTO_DRIVER_LOCK(); 731 cap = crypto_checkdriver(driverid); 732 if (cap != NULL) { 733 driver_finis(cap); 734 err = 0; 735 } else 736 err = EINVAL; 737 CRYPTO_DRIVER_UNLOCK(); 738 739 return err; 740 } 741 742 /* 743 * Clear blockage on a driver. The what parameter indicates whether 744 * the driver is now ready for cryptop's and/or cryptokop's. 745 */ 746 int 747 crypto_unblock(u_int32_t driverid, int what) 748 { 749 struct cryptocap *cap; 750 int err; 751 752 CRYPTO_Q_LOCK(); 753 cap = crypto_checkdriver(driverid); 754 if (cap != NULL) { 755 if (what & CRYPTO_SYMQ) 756 cap->cc_qblocked = 0; 757 if (what & CRYPTO_ASYMQ) 758 cap->cc_kqblocked = 0; 759 if (crp_sleep) 760 wakeup_one(&crp_q); 761 err = 0; 762 } else 763 err = EINVAL; 764 CRYPTO_Q_UNLOCK(); 765 766 return err; 767 } 768 769 /* 770 * Add a crypto request to a queue, to be processed by the kernel thread. 771 */ 772 int 773 crypto_dispatch(struct cryptop *crp) 774 { 775 struct cryptocap *cap; 776 u_int32_t hid; 777 int result; 778 779 cryptostats.cs_ops++; 780 781 #ifdef CRYPTO_TIMING 782 if (crypto_timing) 783 binuptime(&crp->crp_tstamp); 784 #endif 785 786 hid = CRYPTO_SESID2HID(crp->crp_sid); 787 788 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 789 /* 790 * Caller marked the request to be processed 791 * immediately; dispatch it directly to the 792 * driver unless the driver is currently blocked. 793 */ 794 cap = crypto_checkdriver(hid); 795 /* Driver cannot disappeared when there is an active session. */ 796 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 797 if (!cap->cc_qblocked) { 798 result = crypto_invoke(cap, crp, 0); 799 if (result != ERESTART) 800 return (result); 801 /* 802 * The driver ran out of resources, put the request on 803 * the queue. 804 */ 805 } 806 } 807 CRYPTO_Q_LOCK(); 808 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 809 if (crp_sleep) 810 wakeup_one(&crp_q); 811 CRYPTO_Q_UNLOCK(); 812 return 0; 813 } 814 815 /* 816 * Add an asymetric crypto request to a queue, 817 * to be processed by the kernel thread. 818 */ 819 int 820 crypto_kdispatch(struct cryptkop *krp) 821 { 822 int error; 823 824 cryptostats.cs_kops++; 825 826 error = crypto_kinvoke(krp, krp->krp_crid); 827 if (error == ERESTART) { 828 CRYPTO_Q_LOCK(); 829 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 830 if (crp_sleep) 831 wakeup_one(&crp_q); 832 CRYPTO_Q_UNLOCK(); 833 error = 0; 834 } 835 return error; 836 } 837 838 /* 839 * Verify a driver is suitable for the specified operation. 840 */ 841 static __inline int 842 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) 843 { 844 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; 845 } 846 847 /* 848 * Select a driver for an asym operation. The driver must 849 * support the necessary algorithm. The caller can constrain 850 * which device is selected with the flags parameter. The 851 * algorithm we use here is pretty stupid; just use the first 852 * driver that supports the algorithms we need. If there are 853 * multiple suitable drivers we choose the driver with the 854 * fewest active operations. We prefer hardware-backed 855 * drivers to software ones when either may be used. 856 */ 857 static struct cryptocap * 858 crypto_select_kdriver(const struct cryptkop *krp, int flags) 859 { 860 struct cryptocap *cap, *best, *blocked; 861 int match, hid; 862 863 CRYPTO_DRIVER_ASSERT(); 864 865 /* 866 * Look first for hardware crypto devices if permitted. 867 */ 868 if (flags & CRYPTOCAP_F_HARDWARE) 869 match = CRYPTOCAP_F_HARDWARE; 870 else 871 match = CRYPTOCAP_F_SOFTWARE; 872 best = NULL; 873 blocked = NULL; 874 again: 875 for (hid = 0; hid < crypto_drivers_num; hid++) { 876 cap = &crypto_drivers[hid]; 877 /* 878 * If it's not initialized, is in the process of 879 * going away, or is not appropriate (hardware 880 * or software based on match), then skip. 881 */ 882 if (cap->cc_dev == NULL || 883 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 884 (cap->cc_flags & match) == 0) 885 continue; 886 887 /* verify all the algorithms are supported. */ 888 if (kdriver_suitable(cap, krp)) { 889 if (best == NULL || 890 cap->cc_koperations < best->cc_koperations) 891 best = cap; 892 } 893 } 894 if (best != NULL) 895 return best; 896 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 897 /* sort of an Algol 68-style for loop */ 898 match = CRYPTOCAP_F_SOFTWARE; 899 goto again; 900 } 901 return best; 902 } 903 904 /* 905 * Dispatch an assymetric crypto request. 906 */ 907 static int 908 crypto_kinvoke(struct cryptkop *krp, int crid) 909 { 910 struct cryptocap *cap = NULL; 911 int error; 912 913 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 914 KASSERT(krp->krp_callback != NULL, 915 ("%s: krp->crp_callback == NULL", __func__)); 916 917 CRYPTO_DRIVER_LOCK(); 918 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 919 cap = crypto_checkdriver(crid); 920 if (cap != NULL) { 921 /* 922 * Driver present, it must support the necessary 923 * algorithm and, if s/w drivers are excluded, 924 * it must be registered as hardware-backed. 925 */ 926 if (!kdriver_suitable(cap, krp) || 927 (!crypto_devallowsoft && 928 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) 929 cap = NULL; 930 } 931 } else { 932 /* 933 * No requested driver; select based on crid flags. 934 */ 935 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ 936 crid &= ~CRYPTOCAP_F_SOFTWARE; 937 cap = crypto_select_kdriver(krp, crid); 938 } 939 if (cap != NULL && !cap->cc_kqblocked) { 940 krp->krp_hid = cap - crypto_drivers; 941 cap->cc_koperations++; 942 CRYPTO_DRIVER_UNLOCK(); 943 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); 944 CRYPTO_DRIVER_LOCK(); 945 if (error == ERESTART) { 946 cap->cc_koperations--; 947 CRYPTO_DRIVER_UNLOCK(); 948 return (error); 949 } 950 } else { 951 /* 952 * NB: cap is !NULL if device is blocked; in 953 * that case return ERESTART so the operation 954 * is resubmitted if possible. 955 */ 956 error = (cap == NULL) ? ENODEV : ERESTART; 957 } 958 CRYPTO_DRIVER_UNLOCK(); 959 960 if (error) { 961 krp->krp_status = error; 962 crypto_kdone(krp); 963 } 964 return 0; 965 } 966 967 #ifdef CRYPTO_TIMING 968 static void 969 crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 970 { 971 struct bintime now, delta; 972 struct timespec t; 973 uint64_t u; 974 975 binuptime(&now); 976 u = now.frac; 977 delta.frac = now.frac - bt->frac; 978 delta.sec = now.sec - bt->sec; 979 if (u < delta.frac) 980 delta.sec--; 981 bintime2timespec(&delta, &t); 982 timespecadd(&ts->acc, &t); 983 if (timespeccmp(&t, &ts->min, <)) 984 ts->min = t; 985 if (timespeccmp(&t, &ts->max, >)) 986 ts->max = t; 987 ts->count++; 988 989 *bt = now; 990 } 991 #endif 992 993 /* 994 * Dispatch a crypto request to the appropriate crypto devices. 995 */ 996 static int 997 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 998 { 999 1000 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 1001 KASSERT(crp->crp_callback != NULL, 1002 ("%s: crp->crp_callback == NULL", __func__)); 1003 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 1004 1005 #ifdef CRYPTO_TIMING 1006 if (crypto_timing) 1007 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1008 #endif 1009 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 1010 struct cryptodesc *crd; 1011 u_int64_t nid; 1012 1013 /* 1014 * Driver has unregistered; migrate the session and return 1015 * an error to the caller so they'll resubmit the op. 1016 * 1017 * XXX: What if there are more already queued requests for this 1018 * session? 1019 */ 1020 crypto_freesession(crp->crp_sid); 1021 1022 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1023 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1024 1025 /* XXX propagate flags from initial session? */ 1026 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 1027 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) 1028 crp->crp_sid = nid; 1029 1030 crp->crp_etype = EAGAIN; 1031 crypto_done(crp); 1032 return 0; 1033 } else { 1034 /* 1035 * Invoke the driver to process the request. 1036 */ 1037 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); 1038 } 1039 } 1040 1041 /* 1042 * Release a set of crypto descriptors. 1043 */ 1044 void 1045 crypto_freereq(struct cryptop *crp) 1046 { 1047 struct cryptodesc *crd; 1048 1049 if (crp == NULL) 1050 return; 1051 1052 #ifdef DIAGNOSTIC 1053 { 1054 struct cryptop *crp2; 1055 1056 CRYPTO_Q_LOCK(); 1057 TAILQ_FOREACH(crp2, &crp_q, crp_next) { 1058 KASSERT(crp2 != crp, 1059 ("Freeing cryptop from the crypto queue (%p).", 1060 crp)); 1061 } 1062 CRYPTO_Q_UNLOCK(); 1063 CRYPTO_RETQ_LOCK(); 1064 TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { 1065 KASSERT(crp2 != crp, 1066 ("Freeing cryptop from the return queue (%p).", 1067 crp)); 1068 } 1069 CRYPTO_RETQ_UNLOCK(); 1070 } 1071 #endif 1072 1073 while ((crd = crp->crp_desc) != NULL) { 1074 crp->crp_desc = crd->crd_next; 1075 uma_zfree(cryptodesc_zone, crd); 1076 } 1077 uma_zfree(cryptop_zone, crp); 1078 } 1079 1080 /* 1081 * Acquire a set of crypto descriptors. 1082 */ 1083 struct cryptop * 1084 crypto_getreq(int num) 1085 { 1086 struct cryptodesc *crd; 1087 struct cryptop *crp; 1088 1089 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 1090 if (crp != NULL) { 1091 while (num--) { 1092 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 1093 if (crd == NULL) { 1094 crypto_freereq(crp); 1095 return NULL; 1096 } 1097 1098 crd->crd_next = crp->crp_desc; 1099 crp->crp_desc = crd; 1100 } 1101 } 1102 return crp; 1103 } 1104 1105 /* 1106 * Invoke the callback on behalf of the driver. 1107 */ 1108 void 1109 crypto_done(struct cryptop *crp) 1110 { 1111 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 1112 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 1113 crp->crp_flags |= CRYPTO_F_DONE; 1114 if (crp->crp_etype != 0) 1115 cryptostats.cs_errs++; 1116 #ifdef CRYPTO_TIMING 1117 if (crypto_timing) 1118 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 1119 #endif 1120 /* 1121 * CBIMM means unconditionally do the callback immediately; 1122 * CBIFSYNC means do the callback immediately only if the 1123 * operation was done synchronously. Both are used to avoid 1124 * doing extraneous context switches; the latter is mostly 1125 * used with the software crypto driver. 1126 */ 1127 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 1128 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 1129 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 1130 /* 1131 * Do the callback directly. This is ok when the 1132 * callback routine does very little (e.g. the 1133 * /dev/crypto callback method just does a wakeup). 1134 */ 1135 #ifdef CRYPTO_TIMING 1136 if (crypto_timing) { 1137 /* 1138 * NB: We must copy the timestamp before 1139 * doing the callback as the cryptop is 1140 * likely to be reclaimed. 1141 */ 1142 struct bintime t = crp->crp_tstamp; 1143 crypto_tstat(&cryptostats.cs_cb, &t); 1144 crp->crp_callback(crp); 1145 crypto_tstat(&cryptostats.cs_finis, &t); 1146 } else 1147 #endif 1148 crp->crp_callback(crp); 1149 } else { 1150 /* 1151 * Normal case; queue the callback for the thread. 1152 */ 1153 CRYPTO_RETQ_LOCK(); 1154 if (CRYPTO_RETQ_EMPTY()) 1155 wakeup_one(&crp_ret_q); /* shared wait channel */ 1156 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1157 CRYPTO_RETQ_UNLOCK(); 1158 } 1159 } 1160 1161 /* 1162 * Invoke the callback on behalf of the driver. 1163 */ 1164 void 1165 crypto_kdone(struct cryptkop *krp) 1166 { 1167 struct cryptocap *cap; 1168 1169 if (krp->krp_status != 0) 1170 cryptostats.cs_kerrs++; 1171 CRYPTO_DRIVER_LOCK(); 1172 /* XXX: What if driver is loaded in the meantime? */ 1173 if (krp->krp_hid < crypto_drivers_num) { 1174 cap = &crypto_drivers[krp->krp_hid]; 1175 cap->cc_koperations--; 1176 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 1177 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1178 crypto_remove(cap); 1179 } 1180 CRYPTO_DRIVER_UNLOCK(); 1181 CRYPTO_RETQ_LOCK(); 1182 if (CRYPTO_RETQ_EMPTY()) 1183 wakeup_one(&crp_ret_q); /* shared wait channel */ 1184 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1185 CRYPTO_RETQ_UNLOCK(); 1186 } 1187 1188 int 1189 crypto_getfeat(int *featp) 1190 { 1191 int hid, kalg, feat = 0; 1192 1193 CRYPTO_DRIVER_LOCK(); 1194 for (hid = 0; hid < crypto_drivers_num; hid++) { 1195 const struct cryptocap *cap = &crypto_drivers[hid]; 1196 1197 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1198 !crypto_devallowsoft) { 1199 continue; 1200 } 1201 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1202 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) 1203 feat |= 1 << kalg; 1204 } 1205 CRYPTO_DRIVER_UNLOCK(); 1206 *featp = feat; 1207 return (0); 1208 } 1209 1210 /* 1211 * Terminate a thread at module unload. The process that 1212 * initiated this is waiting for us to signal that we're gone; 1213 * wake it up and exit. We use the driver table lock to insure 1214 * we don't do the wakeup before they're waiting. There is no 1215 * race here because the waiter sleeps on the proc lock for the 1216 * thread so it gets notified at the right time because of an 1217 * extra wakeup that's done in exit1(). 1218 */ 1219 static void 1220 crypto_finis(void *chan) 1221 { 1222 CRYPTO_DRIVER_LOCK(); 1223 wakeup_one(chan); 1224 CRYPTO_DRIVER_UNLOCK(); 1225 kthread_exit(0); 1226 } 1227 1228 /* 1229 * Crypto thread, dispatches crypto requests. 1230 */ 1231 static void 1232 crypto_proc(void) 1233 { 1234 struct cryptop *crp, *submit; 1235 struct cryptkop *krp; 1236 struct cryptocap *cap; 1237 u_int32_t hid; 1238 int result, hint; 1239 1240 CRYPTO_Q_LOCK(); 1241 for (;;) { 1242 /* 1243 * Find the first element in the queue that can be 1244 * processed and look-ahead to see if multiple ops 1245 * are ready for the same driver. 1246 */ 1247 submit = NULL; 1248 hint = 0; 1249 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1250 hid = CRYPTO_SESID2HID(crp->crp_sid); 1251 cap = crypto_checkdriver(hid); 1252 /* 1253 * Driver cannot disappeared when there is an active 1254 * session. 1255 */ 1256 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1257 __func__, __LINE__)); 1258 if (cap == NULL || cap->cc_dev == NULL) { 1259 /* Op needs to be migrated, process it. */ 1260 if (submit == NULL) 1261 submit = crp; 1262 break; 1263 } 1264 if (!cap->cc_qblocked) { 1265 if (submit != NULL) { 1266 /* 1267 * We stop on finding another op, 1268 * regardless whether its for the same 1269 * driver or not. We could keep 1270 * searching the queue but it might be 1271 * better to just use a per-driver 1272 * queue instead. 1273 */ 1274 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1275 hint = CRYPTO_HINT_MORE; 1276 break; 1277 } else { 1278 submit = crp; 1279 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1280 break; 1281 /* keep scanning for more are q'd */ 1282 } 1283 } 1284 } 1285 if (submit != NULL) { 1286 TAILQ_REMOVE(&crp_q, submit, crp_next); 1287 hid = CRYPTO_SESID2HID(submit->crp_sid); 1288 cap = crypto_checkdriver(hid); 1289 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1290 __func__, __LINE__)); 1291 result = crypto_invoke(cap, submit, hint); 1292 if (result == ERESTART) { 1293 /* 1294 * The driver ran out of resources, mark the 1295 * driver ``blocked'' for cryptop's and put 1296 * the request back in the queue. It would 1297 * best to put the request back where we got 1298 * it but that's hard so for now we put it 1299 * at the front. This should be ok; putting 1300 * it at the end does not work. 1301 */ 1302 /* XXX validate sid again? */ 1303 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1304 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1305 cryptostats.cs_blocks++; 1306 } 1307 } 1308 1309 /* As above, but for key ops */ 1310 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1311 cap = crypto_checkdriver(krp->krp_hid); 1312 if (cap == NULL || cap->cc_dev == NULL) { 1313 /* 1314 * Operation needs to be migrated, invalidate 1315 * the assigned device so it will reselect a 1316 * new one below. Propagate the original 1317 * crid selection flags if supplied. 1318 */ 1319 krp->krp_hid = krp->krp_crid & 1320 (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); 1321 if (krp->krp_hid == 0) 1322 krp->krp_hid = 1323 CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; 1324 break; 1325 } 1326 if (!cap->cc_kqblocked) 1327 break; 1328 } 1329 if (krp != NULL) { 1330 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1331 result = crypto_kinvoke(krp, krp->krp_hid); 1332 if (result == ERESTART) { 1333 /* 1334 * The driver ran out of resources, mark the 1335 * driver ``blocked'' for cryptkop's and put 1336 * the request back in the queue. It would 1337 * best to put the request back where we got 1338 * it but that's hard so for now we put it 1339 * at the front. This should be ok; putting 1340 * it at the end does not work. 1341 */ 1342 /* XXX validate sid again? */ 1343 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1344 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1345 cryptostats.cs_kblocks++; 1346 } 1347 } 1348 1349 if (submit == NULL && krp == NULL) { 1350 /* 1351 * Nothing more to be processed. Sleep until we're 1352 * woken because there are more ops to process. 1353 * This happens either by submission or by a driver 1354 * becoming unblocked and notifying us through 1355 * crypto_unblock. Note that when we wakeup we 1356 * start processing each queue again from the 1357 * front. It's not clear that it's important to 1358 * preserve this ordering since ops may finish 1359 * out of order if dispatched to different devices 1360 * and some become blocked while others do not. 1361 */ 1362 crp_sleep = 1; 1363 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1364 crp_sleep = 0; 1365 if (cryptoproc == NULL) 1366 break; 1367 cryptostats.cs_intrs++; 1368 } 1369 } 1370 CRYPTO_Q_UNLOCK(); 1371 1372 crypto_finis(&crp_q); 1373 } 1374 1375 /* 1376 * Crypto returns thread, does callbacks for processed crypto requests. 1377 * Callbacks are done here, rather than in the crypto drivers, because 1378 * callbacks typically are expensive and would slow interrupt handling. 1379 */ 1380 static void 1381 crypto_ret_proc(void) 1382 { 1383 struct cryptop *crpt; 1384 struct cryptkop *krpt; 1385 1386 CRYPTO_RETQ_LOCK(); 1387 for (;;) { 1388 /* Harvest return q's for completed ops */ 1389 crpt = TAILQ_FIRST(&crp_ret_q); 1390 if (crpt != NULL) 1391 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1392 1393 krpt = TAILQ_FIRST(&crp_ret_kq); 1394 if (krpt != NULL) 1395 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1396 1397 if (crpt != NULL || krpt != NULL) { 1398 CRYPTO_RETQ_UNLOCK(); 1399 /* 1400 * Run callbacks unlocked. 1401 */ 1402 if (crpt != NULL) { 1403 #ifdef CRYPTO_TIMING 1404 if (crypto_timing) { 1405 /* 1406 * NB: We must copy the timestamp before 1407 * doing the callback as the cryptop is 1408 * likely to be reclaimed. 1409 */ 1410 struct bintime t = crpt->crp_tstamp; 1411 crypto_tstat(&cryptostats.cs_cb, &t); 1412 crpt->crp_callback(crpt); 1413 crypto_tstat(&cryptostats.cs_finis, &t); 1414 } else 1415 #endif 1416 crpt->crp_callback(crpt); 1417 } 1418 if (krpt != NULL) 1419 krpt->krp_callback(krpt); 1420 CRYPTO_RETQ_LOCK(); 1421 } else { 1422 /* 1423 * Nothing more to be processed. Sleep until we're 1424 * woken because there are more returns to process. 1425 */ 1426 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1427 "crypto_ret_wait", 0); 1428 if (cryptoretproc == NULL) 1429 break; 1430 cryptostats.cs_rets++; 1431 } 1432 } 1433 CRYPTO_RETQ_UNLOCK(); 1434 1435 crypto_finis(&crp_ret_q); 1436 } 1437 1438 #ifdef DDB 1439 static void 1440 db_show_drivers(void) 1441 { 1442 int hid; 1443 1444 db_printf("%12s %4s %4s %8s %2s %2s\n" 1445 , "Device" 1446 , "Ses" 1447 , "Kops" 1448 , "Flags" 1449 , "QB" 1450 , "KB" 1451 ); 1452 for (hid = 0; hid < crypto_drivers_num; hid++) { 1453 const struct cryptocap *cap = &crypto_drivers[hid]; 1454 if (cap->cc_dev == NULL) 1455 continue; 1456 db_printf("%-12s %4u %4u %08x %2u %2u\n" 1457 , device_get_nameunit(cap->cc_dev) 1458 , cap->cc_sessions 1459 , cap->cc_koperations 1460 , cap->cc_flags 1461 , cap->cc_qblocked 1462 , cap->cc_kqblocked 1463 ); 1464 } 1465 } 1466 1467 DB_SHOW_COMMAND(crypto, db_show_crypto) 1468 { 1469 struct cryptop *crp; 1470 1471 db_show_drivers(); 1472 db_printf("\n"); 1473 1474 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", 1475 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", 1476 "Desc", "Callback"); 1477 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1478 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" 1479 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1480 , (int) CRYPTO_SESID2CAPS(crp->crp_sid) 1481 , crp->crp_ilen, crp->crp_olen 1482 , crp->crp_etype 1483 , crp->crp_flags 1484 , crp->crp_desc 1485 , crp->crp_callback 1486 ); 1487 } 1488 if (!TAILQ_EMPTY(&crp_ret_q)) { 1489 db_printf("\n%4s %4s %4s %8s\n", 1490 "HID", "Etype", "Flags", "Callback"); 1491 TAILQ_FOREACH(crp, &crp_ret_q, crp_next) { 1492 db_printf("%4u %4u %04x %8p\n" 1493 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1494 , crp->crp_etype 1495 , crp->crp_flags 1496 , crp->crp_callback 1497 ); 1498 } 1499 } 1500 } 1501 1502 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) 1503 { 1504 struct cryptkop *krp; 1505 1506 db_show_drivers(); 1507 db_printf("\n"); 1508 1509 db_printf("%4s %5s %4s %4s %8s %4s %8s\n", 1510 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); 1511 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1512 db_printf("%4u %5u %4u %4u %08x %4u %8p\n" 1513 , krp->krp_op 1514 , krp->krp_status 1515 , krp->krp_iparams, krp->krp_oparams 1516 , krp->krp_crid, krp->krp_hid 1517 , krp->krp_callback 1518 ); 1519 } 1520 if (!TAILQ_EMPTY(&crp_ret_q)) { 1521 db_printf("%4s %5s %8s %4s %8s\n", 1522 "Op", "Status", "CRID", "HID", "Callback"); 1523 TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) { 1524 db_printf("%4u %5u %08x %4u %8p\n" 1525 , krp->krp_op 1526 , krp->krp_status 1527 , krp->krp_crid, krp->krp_hid 1528 , krp->krp_callback 1529 ); 1530 } 1531 } 1532 } 1533 #endif 1534 1535 int crypto_modevent(module_t mod, int type, void *unused); 1536 1537 /* 1538 * Initialization code, both for static and dynamic loading. 1539 * Note this is not invoked with the usual MODULE_DECLARE 1540 * mechanism but instead is listed as a dependency by the 1541 * cryptosoft driver. This guarantees proper ordering of 1542 * calls on module load/unload. 1543 */ 1544 int 1545 crypto_modevent(module_t mod, int type, void *unused) 1546 { 1547 int error = EINVAL; 1548 1549 switch (type) { 1550 case MOD_LOAD: 1551 error = crypto_init(); 1552 if (error == 0 && bootverbose) 1553 printf("crypto: <crypto core>\n"); 1554 break; 1555 case MOD_UNLOAD: 1556 /*XXX disallow if active sessions */ 1557 error = 0; 1558 crypto_destroy(); 1559 return 0; 1560 } 1561 return error; 1562 } 1563 MODULE_VERSION(crypto, 1); 1564 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
Cache object: f5bab669ce78caaf36c2c6c55a343255
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]