Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 2 /*- 3 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 4 * 5 * This code was written by Angelos D. Keromytis in Athens, Greece, in 6 * February 2000. Network Security Technologies Inc. (NSTI) kindly 7 * supported the development of this code. 8 * 9 * Copyright (c) 2000, 2001 Angelos D. Keromytis 10 * 11 * Permission to use, copy, and modify this software with or without fee 12 * is hereby granted, provided that this entire notice is included in 13 * all source code copies of any software which is or includes a copy or 14 * modification of this software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 17 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 18 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 19 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 20 * PURPOSE. 21 */ 22 23 #include <sys/cdefs.h> 24 __FBSDID("$FreeBSD: releng/6.1/sys/opencrypto/crypto.c 156307 2006-03-05 00:48:05Z wkoszek $"); 25 26 #define CRYPTO_TIMING /* enable timing support */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/eventhandler.h> 31 #include <sys/kernel.h> 32 #include <sys/kthread.h> 33 #include <sys/lock.h> 34 #include <sys/module.h> 35 #include <sys/mutex.h> 36 #include <sys/malloc.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 40 #include <vm/uma.h> 41 #include <opencrypto/cryptodev.h> 42 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 43 44 /* 45 * Crypto drivers register themselves by allocating a slot in the 46 * crypto_drivers table with crypto_get_driverid() and then registering 47 * each algorithm they support with crypto_register() and crypto_kregister(). 48 */ 49 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 50 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 51 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 52 static struct cryptocap *crypto_drivers = NULL; 53 static int crypto_drivers_num = 0; 54 55 /* 56 * There are two queues for crypto requests; one for symmetric (e.g. 57 * cipher) operations and one for asymmetric (e.g. MOD)operations. 58 * A single mutex is used to lock access to both queues. We could 59 * have one per-queue but having one simplifies handling of block/unblock 60 * operations. 61 */ 62 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 63 static TAILQ_HEAD(,cryptkop) crp_kq; 64 static struct mtx crypto_q_mtx; 65 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 66 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 67 68 /* 69 * There are two queues for processing completed crypto requests; one 70 * for the symmetric and one for the asymmetric ops. We only need one 71 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 72 * mutex is used to lock access to both queues. Note that this lock 73 * must be separate from the lock on request queues to insure driver 74 * callbacks don't generate lock order reversals. 75 */ 76 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 77 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 78 static struct mtx crypto_ret_q_mtx; 79 #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 80 #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 81 82 static uma_zone_t cryptop_zone; 83 static uma_zone_t cryptodesc_zone; 84 85 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 86 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 87 &crypto_userasymcrypto, 0, 88 "Enable/disable user-mode access to asymmetric crypto support"); 89 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 90 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 91 &crypto_devallowsoft, 0, 92 "Enable/disable use of software asym crypto support"); 93 94 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 95 96 static void crypto_proc(void); 97 static struct proc *cryptoproc; 98 static void crypto_ret_proc(void); 99 static struct proc *cryptoretproc; 100 static void crypto_destroy(void); 101 static int crypto_invoke(struct cryptop *crp, int hint); 102 static int crypto_kinvoke(struct cryptkop *krp, int hint); 103 104 static struct cryptostats cryptostats; 105 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 106 cryptostats, "Crypto system statistics"); 107 108 #ifdef CRYPTO_TIMING 109 static int crypto_timing = 0; 110 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 111 &crypto_timing, 0, "Enable/disable crypto timing support"); 112 #endif 113 114 static int 115 crypto_init(void) 116 { 117 int error; 118 119 mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 120 MTX_DEF|MTX_QUIET); 121 122 TAILQ_INIT(&crp_q); 123 TAILQ_INIT(&crp_kq); 124 mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 125 126 TAILQ_INIT(&crp_ret_q); 127 TAILQ_INIT(&crp_ret_kq); 128 mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 129 130 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 131 0, 0, 0, 0, 132 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 133 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 134 0, 0, 0, 0, 135 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 136 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 137 printf("crypto_init: cannot setup crypto zones\n"); 138 error = ENOMEM; 139 goto bad; 140 } 141 142 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 143 crypto_drivers = malloc(crypto_drivers_num * 144 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 145 if (crypto_drivers == NULL) { 146 printf("crypto_init: cannot setup crypto drivers\n"); 147 error = ENOMEM; 148 goto bad; 149 } 150 151 error = kthread_create((void (*)(void *)) crypto_proc, NULL, 152 &cryptoproc, 0, 0, "crypto"); 153 if (error) { 154 printf("crypto_init: cannot start crypto thread; error %d", 155 error); 156 goto bad; 157 } 158 159 error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 160 &cryptoretproc, 0, 0, "crypto returns"); 161 if (error) { 162 printf("crypto_init: cannot start cryptoret thread; error %d", 163 error); 164 goto bad; 165 } 166 return 0; 167 bad: 168 crypto_destroy(); 169 return error; 170 } 171 172 /* 173 * Signal a crypto thread to terminate. We use the driver 174 * table lock to synchronize the sleep/wakeups so that we 175 * are sure the threads have terminated before we release 176 * the data structures they use. See crypto_finis below 177 * for the other half of this song-and-dance. 178 */ 179 static void 180 crypto_terminate(struct proc **pp, void *q) 181 { 182 struct proc *p; 183 184 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 185 p = *pp; 186 *pp = NULL; 187 if (p) { 188 wakeup_one(q); 189 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 190 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 191 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 192 PROC_UNLOCK(p); 193 CRYPTO_DRIVER_LOCK(); 194 } 195 } 196 197 static void 198 crypto_destroy(void) 199 { 200 /* 201 * Terminate any crypto threads. 202 */ 203 CRYPTO_DRIVER_LOCK(); 204 crypto_terminate(&cryptoproc, &crp_q); 205 crypto_terminate(&cryptoretproc, &crp_ret_q); 206 CRYPTO_DRIVER_UNLOCK(); 207 208 /* XXX flush queues??? */ 209 210 /* 211 * Reclaim dynamically allocated resources. 212 */ 213 if (crypto_drivers != NULL) 214 free(crypto_drivers, M_CRYPTO_DATA); 215 216 if (cryptodesc_zone != NULL) 217 uma_zdestroy(cryptodesc_zone); 218 if (cryptop_zone != NULL) 219 uma_zdestroy(cryptop_zone); 220 mtx_destroy(&crypto_q_mtx); 221 mtx_destroy(&crypto_ret_q_mtx); 222 mtx_destroy(&crypto_drivers_mtx); 223 } 224 225 /* 226 * Initialization code, both for static and dynamic loading. 227 */ 228 static int 229 crypto_modevent(module_t mod, int type, void *unused) 230 { 231 int error = EINVAL; 232 233 switch (type) { 234 case MOD_LOAD: 235 error = crypto_init(); 236 if (error == 0 && bootverbose) 237 printf("crypto: <crypto core>\n"); 238 break; 239 case MOD_UNLOAD: 240 /*XXX disallow if active sessions */ 241 error = 0; 242 crypto_destroy(); 243 return 0; 244 } 245 return error; 246 } 247 248 static moduledata_t crypto_mod = { 249 "crypto", 250 crypto_modevent, 251 0 252 }; 253 MODULE_VERSION(crypto, 1); 254 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 255 MODULE_DEPEND(crypto, zlib, 1, 1, 1); 256 257 /* 258 * Create a new session. 259 */ 260 int 261 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 262 { 263 struct cryptoini *cr; 264 u_int32_t hid, lid; 265 int err = EINVAL; 266 267 CRYPTO_DRIVER_LOCK(); 268 269 if (crypto_drivers == NULL) 270 goto done; 271 272 /* 273 * The algorithm we use here is pretty stupid; just use the 274 * first driver that supports all the algorithms we need. 275 * 276 * XXX We need more smarts here (in real life too, but that's 277 * XXX another story altogether). 278 */ 279 280 for (hid = 0; hid < crypto_drivers_num; hid++) { 281 struct cryptocap *cap = &crypto_drivers[hid]; 282 /* 283 * If it's not initialized or has remaining sessions 284 * referencing it, skip. 285 */ 286 if (cap->cc_newsession == NULL || 287 (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) 288 continue; 289 290 /* Hardware required -- ignore software drivers. */ 291 if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) 292 continue; 293 /* Software required -- ignore hardware drivers. */ 294 if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 295 continue; 296 297 /* See if all the algorithms are supported. */ 298 for (cr = cri; cr; cr = cr->cri_next) 299 if (cap->cc_alg[cr->cri_alg] == 0) 300 break; 301 302 if (cr == NULL) { 303 /* Ok, all algorithms are supported. */ 304 305 /* 306 * Can't do everything in one session. 307 * 308 * XXX Fix this. We need to inject a "virtual" session layer right 309 * XXX about here. 310 */ 311 312 /* Call the driver initialization routine. */ 313 lid = hid; /* Pass the driver ID. */ 314 err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri); 315 if (err == 0) { 316 /* XXX assert (hid &~ 0xffffff) == 0 */ 317 /* XXX assert (cap->cc_flags &~ 0xff) == 0 */ 318 (*sid) = ((cap->cc_flags & 0xff) << 24) | hid; 319 (*sid) <<= 32; 320 (*sid) |= (lid & 0xffffffff); 321 cap->cc_sessions++; 322 } 323 break; 324 } 325 } 326 done: 327 CRYPTO_DRIVER_UNLOCK(); 328 return err; 329 } 330 331 /* 332 * Delete an existing session (or a reserved session on an unregistered 333 * driver). 334 */ 335 int 336 crypto_freesession(u_int64_t sid) 337 { 338 u_int32_t hid; 339 int err; 340 341 CRYPTO_DRIVER_LOCK(); 342 343 if (crypto_drivers == NULL) { 344 err = EINVAL; 345 goto done; 346 } 347 348 /* Determine two IDs. */ 349 hid = CRYPTO_SESID2HID(sid); 350 351 if (hid >= crypto_drivers_num) { 352 err = ENOENT; 353 goto done; 354 } 355 356 if (crypto_drivers[hid].cc_sessions) 357 crypto_drivers[hid].cc_sessions--; 358 359 /* Call the driver cleanup routine, if available. */ 360 if (crypto_drivers[hid].cc_freesession) 361 err = crypto_drivers[hid].cc_freesession( 362 crypto_drivers[hid].cc_arg, sid); 363 else 364 err = 0; 365 366 /* 367 * If this was the last session of a driver marked as invalid, 368 * make the entry available for reuse. 369 */ 370 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 371 crypto_drivers[hid].cc_sessions == 0) 372 bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 373 374 done: 375 CRYPTO_DRIVER_UNLOCK(); 376 return err; 377 } 378 379 /* 380 * Return an unused driver id. Used by drivers prior to registering 381 * support for the algorithms they handle. 382 */ 383 int32_t 384 crypto_get_driverid(u_int32_t flags) 385 { 386 struct cryptocap *newdrv; 387 int i; 388 389 CRYPTO_DRIVER_LOCK(); 390 391 for (i = 0; i < crypto_drivers_num; i++) 392 if (crypto_drivers[i].cc_process == NULL && 393 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 394 crypto_drivers[i].cc_sessions == 0) 395 break; 396 397 /* Out of entries, allocate some more. */ 398 if (i == crypto_drivers_num) { 399 /* Be careful about wrap-around. */ 400 if (2 * crypto_drivers_num <= crypto_drivers_num) { 401 CRYPTO_DRIVER_UNLOCK(); 402 printf("crypto: driver count wraparound!\n"); 403 return -1; 404 } 405 406 newdrv = malloc(2 * crypto_drivers_num * 407 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 408 if (newdrv == NULL) { 409 CRYPTO_DRIVER_UNLOCK(); 410 printf("crypto: no space to expand driver table!\n"); 411 return -1; 412 } 413 414 bcopy(crypto_drivers, newdrv, 415 crypto_drivers_num * sizeof(struct cryptocap)); 416 417 crypto_drivers_num *= 2; 418 419 free(crypto_drivers, M_CRYPTO_DATA); 420 crypto_drivers = newdrv; 421 } 422 423 /* NB: state is zero'd on free */ 424 crypto_drivers[i].cc_sessions = 1; /* Mark */ 425 crypto_drivers[i].cc_flags = flags; 426 if (bootverbose) 427 printf("crypto: assign driver %u, flags %u\n", i, flags); 428 429 CRYPTO_DRIVER_UNLOCK(); 430 431 return i; 432 } 433 434 static struct cryptocap * 435 crypto_checkdriver(u_int32_t hid) 436 { 437 if (crypto_drivers == NULL) 438 return NULL; 439 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 440 } 441 442 /* 443 * Register support for a key-related algorithm. This routine 444 * is called once for each algorithm supported a driver. 445 */ 446 int 447 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 448 int (*kprocess)(void*, struct cryptkop *, int), 449 void *karg) 450 { 451 struct cryptocap *cap; 452 int err; 453 454 CRYPTO_DRIVER_LOCK(); 455 456 cap = crypto_checkdriver(driverid); 457 if (cap != NULL && 458 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 459 /* 460 * XXX Do some performance testing to determine placing. 461 * XXX We probably need an auxiliary data structure that 462 * XXX describes relative performances. 463 */ 464 465 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 466 if (bootverbose) 467 printf("crypto: driver %u registers key alg %u flags %u\n" 468 , driverid 469 , kalg 470 , flags 471 ); 472 473 if (cap->cc_kprocess == NULL) { 474 cap->cc_karg = karg; 475 cap->cc_kprocess = kprocess; 476 } 477 err = 0; 478 } else 479 err = EINVAL; 480 481 CRYPTO_DRIVER_UNLOCK(); 482 return err; 483 } 484 485 /* 486 * Register support for a non-key-related algorithm. This routine 487 * is called once for each such algorithm supported by a driver. 488 */ 489 int 490 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 491 u_int32_t flags, 492 int (*newses)(void*, u_int32_t*, struct cryptoini*), 493 int (*freeses)(void*, u_int64_t), 494 int (*process)(void*, struct cryptop *, int), 495 void *arg) 496 { 497 struct cryptocap *cap; 498 int err; 499 500 CRYPTO_DRIVER_LOCK(); 501 502 cap = crypto_checkdriver(driverid); 503 /* NB: algorithms are in the range [1..max] */ 504 if (cap != NULL && 505 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 506 /* 507 * XXX Do some performance testing to determine placing. 508 * XXX We probably need an auxiliary data structure that 509 * XXX describes relative performances. 510 */ 511 512 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 513 cap->cc_max_op_len[alg] = maxoplen; 514 if (bootverbose) 515 printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 516 , driverid 517 , alg 518 , flags 519 , maxoplen 520 ); 521 522 if (cap->cc_process == NULL) { 523 cap->cc_arg = arg; 524 cap->cc_newsession = newses; 525 cap->cc_process = process; 526 cap->cc_freesession = freeses; 527 cap->cc_sessions = 0; /* Unmark */ 528 } 529 err = 0; 530 } else 531 err = EINVAL; 532 533 CRYPTO_DRIVER_UNLOCK(); 534 return err; 535 } 536 537 /* 538 * Unregister a crypto driver. If there are pending sessions using it, 539 * leave enough information around so that subsequent calls using those 540 * sessions will correctly detect the driver has been unregistered and 541 * reroute requests. 542 */ 543 int 544 crypto_unregister(u_int32_t driverid, int alg) 545 { 546 int i, err; 547 u_int32_t ses; 548 struct cryptocap *cap; 549 550 CRYPTO_DRIVER_LOCK(); 551 552 cap = crypto_checkdriver(driverid); 553 if (cap != NULL && 554 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 555 cap->cc_alg[alg] != 0) { 556 cap->cc_alg[alg] = 0; 557 cap->cc_max_op_len[alg] = 0; 558 559 /* Was this the last algorithm ? */ 560 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 561 if (cap->cc_alg[i] != 0) 562 break; 563 564 if (i == CRYPTO_ALGORITHM_MAX + 1) { 565 ses = cap->cc_sessions; 566 bzero(cap, sizeof(struct cryptocap)); 567 if (ses != 0) { 568 /* 569 * If there are pending sessions, just mark as invalid. 570 */ 571 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 572 cap->cc_sessions = ses; 573 } 574 } 575 err = 0; 576 } else 577 err = EINVAL; 578 579 CRYPTO_DRIVER_UNLOCK(); 580 return err; 581 } 582 583 /* 584 * Unregister all algorithms associated with a crypto driver. 585 * If there are pending sessions using it, leave enough information 586 * around so that subsequent calls using those sessions will 587 * correctly detect the driver has been unregistered and reroute 588 * requests. 589 */ 590 int 591 crypto_unregister_all(u_int32_t driverid) 592 { 593 int i, err; 594 u_int32_t ses; 595 struct cryptocap *cap; 596 597 CRYPTO_DRIVER_LOCK(); 598 599 cap = crypto_checkdriver(driverid); 600 if (cap != NULL) { 601 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 602 cap->cc_alg[i] = 0; 603 cap->cc_max_op_len[i] = 0; 604 } 605 ses = cap->cc_sessions; 606 bzero(cap, sizeof(struct cryptocap)); 607 if (ses != 0) { 608 /* 609 * If there are pending sessions, just mark as invalid. 610 */ 611 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 612 cap->cc_sessions = ses; 613 } 614 err = 0; 615 } else 616 err = EINVAL; 617 618 CRYPTO_DRIVER_UNLOCK(); 619 return err; 620 } 621 622 /* 623 * Clear blockage on a driver. The what parameter indicates whether 624 * the driver is now ready for cryptop's and/or cryptokop's. 625 */ 626 int 627 crypto_unblock(u_int32_t driverid, int what) 628 { 629 struct cryptocap *cap; 630 int needwakeup, err; 631 632 CRYPTO_Q_LOCK(); 633 cap = crypto_checkdriver(driverid); 634 if (cap != NULL) { 635 needwakeup = 0; 636 if (what & CRYPTO_SYMQ) { 637 needwakeup |= cap->cc_qblocked; 638 cap->cc_qblocked = 0; 639 } 640 if (what & CRYPTO_ASYMQ) { 641 needwakeup |= cap->cc_kqblocked; 642 cap->cc_kqblocked = 0; 643 } 644 if (needwakeup) 645 wakeup_one(&crp_q); 646 err = 0; 647 } else 648 err = EINVAL; 649 CRYPTO_Q_UNLOCK(); 650 651 return err; 652 } 653 654 /* 655 * Add a crypto request to a queue, to be processed by the kernel thread. 656 */ 657 int 658 crypto_dispatch(struct cryptop *crp) 659 { 660 u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid); 661 int result; 662 663 cryptostats.cs_ops++; 664 665 #ifdef CRYPTO_TIMING 666 if (crypto_timing) 667 binuptime(&crp->crp_tstamp); 668 #endif 669 670 CRYPTO_Q_LOCK(); 671 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 672 struct cryptocap *cap; 673 /* 674 * Caller marked the request to be processed 675 * immediately; dispatch it directly to the 676 * driver unless the driver is currently blocked. 677 */ 678 cap = crypto_checkdriver(hid); 679 if (cap && !cap->cc_qblocked) { 680 result = crypto_invoke(crp, 0); 681 if (result == ERESTART) { 682 /* 683 * The driver ran out of resources, mark the 684 * driver ``blocked'' for cryptop's and put 685 * the request on the queue. 686 * 687 * XXX ops are placed at the tail so their 688 * order is preserved but this can place them 689 * behind batch'd ops. 690 */ 691 crypto_drivers[hid].cc_qblocked = 1; 692 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 693 cryptostats.cs_blocks++; 694 result = 0; 695 } 696 } else { 697 /* 698 * The driver is blocked, just queue the op until 699 * it unblocks and the kernel thread gets kicked. 700 */ 701 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 702 result = 0; 703 } 704 } else { 705 int wasempty; 706 /* 707 * Caller marked the request as ``ok to delay''; 708 * queue it for the dispatch thread. This is desirable 709 * when the operation is low priority and/or suitable 710 * for batching. 711 */ 712 wasempty = TAILQ_EMPTY(&crp_q); 713 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 714 if (wasempty) 715 wakeup_one(&crp_q); 716 result = 0; 717 } 718 CRYPTO_Q_UNLOCK(); 719 720 return result; 721 } 722 723 /* 724 * Add an asymetric crypto request to a queue, 725 * to be processed by the kernel thread. 726 */ 727 int 728 crypto_kdispatch(struct cryptkop *krp) 729 { 730 struct cryptocap *cap; 731 int result; 732 733 cryptostats.cs_kops++; 734 735 CRYPTO_Q_LOCK(); 736 cap = crypto_checkdriver(krp->krp_hid); 737 if (cap && !cap->cc_kqblocked) { 738 result = crypto_kinvoke(krp, 0); 739 if (result == ERESTART) { 740 /* 741 * The driver ran out of resources, mark the 742 * driver ``blocked'' for cryptkop's and put 743 * the request back in the queue. It would 744 * best to put the request back where we got 745 * it but that's hard so for now we put it 746 * at the front. This should be ok; putting 747 * it at the end does not work. 748 */ 749 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 750 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 751 cryptostats.cs_kblocks++; 752 } 753 } else { 754 /* 755 * The driver is blocked, just queue the op until 756 * it unblocks and the kernel thread gets kicked. 757 */ 758 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 759 result = 0; 760 } 761 CRYPTO_Q_UNLOCK(); 762 763 return result; 764 } 765 766 /* 767 * Dispatch an assymetric crypto request to the appropriate crypto devices. 768 */ 769 static int 770 crypto_kinvoke(struct cryptkop *krp, int hint) 771 { 772 u_int32_t hid; 773 int error; 774 775 mtx_assert(&crypto_q_mtx, MA_OWNED); 776 777 /* Sanity checks. */ 778 if (krp == NULL) 779 return EINVAL; 780 if (krp->krp_callback == NULL) { 781 free(krp, M_XDATA); /* XXX allocated in cryptodev */ 782 return EINVAL; 783 } 784 785 for (hid = 0; hid < crypto_drivers_num; hid++) { 786 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 787 !crypto_devallowsoft) 788 continue; 789 if (crypto_drivers[hid].cc_kprocess == NULL) 790 continue; 791 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 792 CRYPTO_ALG_FLAG_SUPPORTED) == 0) 793 continue; 794 break; 795 } 796 if (hid < crypto_drivers_num) { 797 krp->krp_hid = hid; 798 error = crypto_drivers[hid].cc_kprocess( 799 crypto_drivers[hid].cc_karg, krp, hint); 800 } else 801 error = ENODEV; 802 803 if (error) { 804 krp->krp_status = error; 805 crypto_kdone(krp); 806 } 807 return 0; 808 } 809 810 #ifdef CRYPTO_TIMING 811 static void 812 crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 813 { 814 struct bintime now, delta; 815 struct timespec t; 816 uint64_t u; 817 818 binuptime(&now); 819 u = now.frac; 820 delta.frac = now.frac - bt->frac; 821 delta.sec = now.sec - bt->sec; 822 if (u < delta.frac) 823 delta.sec--; 824 bintime2timespec(&delta, &t); 825 timespecadd(&ts->acc, &t); 826 if (timespeccmp(&t, &ts->min, <)) 827 ts->min = t; 828 if (timespeccmp(&t, &ts->max, >)) 829 ts->max = t; 830 ts->count++; 831 832 *bt = now; 833 } 834 #endif 835 836 /* 837 * Dispatch a crypto request to the appropriate crypto devices. 838 */ 839 static int 840 crypto_invoke(struct cryptop *crp, int hint) 841 { 842 u_int32_t hid; 843 int (*process)(void*, struct cryptop *, int); 844 845 #ifdef CRYPTO_TIMING 846 if (crypto_timing) 847 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 848 #endif 849 /* Sanity checks. */ 850 if (crp == NULL) 851 return EINVAL; 852 if (crp->crp_callback == NULL) { 853 crypto_freereq(crp); 854 return EINVAL; 855 } 856 if (crp->crp_desc == NULL) { 857 crp->crp_etype = EINVAL; 858 crypto_done(crp); 859 return 0; 860 } 861 862 hid = CRYPTO_SESID2HID(crp->crp_sid); 863 if (hid < crypto_drivers_num) { 864 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 865 crypto_freesession(crp->crp_sid); 866 process = crypto_drivers[hid].cc_process; 867 } else { 868 process = NULL; 869 } 870 871 if (process == NULL) { 872 struct cryptodesc *crd; 873 u_int64_t nid; 874 875 /* 876 * Driver has unregistered; migrate the session and return 877 * an error to the caller so they'll resubmit the op. 878 */ 879 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 880 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 881 882 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 883 crp->crp_sid = nid; 884 885 crp->crp_etype = EAGAIN; 886 crypto_done(crp); 887 return 0; 888 } else { 889 /* 890 * Invoke the driver to process the request. 891 */ 892 return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 893 } 894 } 895 896 /* 897 * Release a set of crypto descriptors. 898 */ 899 void 900 crypto_freereq(struct cryptop *crp) 901 { 902 struct cryptodesc *crd; 903 904 if (crp == NULL) 905 return; 906 907 while ((crd = crp->crp_desc) != NULL) { 908 crp->crp_desc = crd->crd_next; 909 uma_zfree(cryptodesc_zone, crd); 910 } 911 912 uma_zfree(cryptop_zone, crp); 913 } 914 915 /* 916 * Acquire a set of crypto descriptors. 917 */ 918 struct cryptop * 919 crypto_getreq(int num) 920 { 921 struct cryptodesc *crd; 922 struct cryptop *crp; 923 924 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 925 if (crp != NULL) { 926 while (num--) { 927 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 928 if (crd == NULL) { 929 crypto_freereq(crp); 930 return NULL; 931 } 932 933 crd->crd_next = crp->crp_desc; 934 crp->crp_desc = crd; 935 } 936 } 937 return crp; 938 } 939 940 /* 941 * Invoke the callback on behalf of the driver. 942 */ 943 void 944 crypto_done(struct cryptop *crp) 945 { 946 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 947 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 948 crp->crp_flags |= CRYPTO_F_DONE; 949 if (crp->crp_etype != 0) 950 cryptostats.cs_errs++; 951 #ifdef CRYPTO_TIMING 952 if (crypto_timing) 953 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 954 #endif 955 /* 956 * CBIMM means unconditionally do the callback immediately; 957 * CBIFSYNC means do the callback immediately only if the 958 * operation was done synchronously. Both are used to avoid 959 * doing extraneous context switches; the latter is mostly 960 * used with the software crypto driver. 961 */ 962 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 963 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 964 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 965 /* 966 * Do the callback directly. This is ok when the 967 * callback routine does very little (e.g. the 968 * /dev/crypto callback method just does a wakeup). 969 */ 970 #ifdef CRYPTO_TIMING 971 if (crypto_timing) { 972 /* 973 * NB: We must copy the timestamp before 974 * doing the callback as the cryptop is 975 * likely to be reclaimed. 976 */ 977 struct bintime t = crp->crp_tstamp; 978 crypto_tstat(&cryptostats.cs_cb, &t); 979 crp->crp_callback(crp); 980 crypto_tstat(&cryptostats.cs_finis, &t); 981 } else 982 #endif 983 crp->crp_callback(crp); 984 } else { 985 int wasempty; 986 /* 987 * Normal case; queue the callback for the thread. 988 */ 989 CRYPTO_RETQ_LOCK(); 990 wasempty = TAILQ_EMPTY(&crp_ret_q); 991 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 992 993 if (wasempty) 994 wakeup_one(&crp_ret_q); /* shared wait channel */ 995 CRYPTO_RETQ_UNLOCK(); 996 } 997 } 998 999 /* 1000 * Invoke the callback on behalf of the driver. 1001 */ 1002 void 1003 crypto_kdone(struct cryptkop *krp) 1004 { 1005 int wasempty; 1006 1007 if (krp->krp_status != 0) 1008 cryptostats.cs_kerrs++; 1009 CRYPTO_RETQ_LOCK(); 1010 wasempty = TAILQ_EMPTY(&crp_ret_kq); 1011 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1012 1013 if (wasempty) 1014 wakeup_one(&crp_ret_q); /* shared wait channel */ 1015 CRYPTO_RETQ_UNLOCK(); 1016 } 1017 1018 int 1019 crypto_getfeat(int *featp) 1020 { 1021 int hid, kalg, feat = 0; 1022 1023 if (!crypto_userasymcrypto) 1024 goto out; 1025 1026 CRYPTO_DRIVER_LOCK(); 1027 for (hid = 0; hid < crypto_drivers_num; hid++) { 1028 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 1029 !crypto_devallowsoft) { 1030 continue; 1031 } 1032 if (crypto_drivers[hid].cc_kprocess == NULL) 1033 continue; 1034 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1035 if ((crypto_drivers[hid].cc_kalg[kalg] & 1036 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1037 feat |= 1 << kalg; 1038 } 1039 CRYPTO_DRIVER_UNLOCK(); 1040 out: 1041 *featp = feat; 1042 return (0); 1043 } 1044 1045 /* 1046 * Terminate a thread at module unload. The process that 1047 * initiated this is waiting for us to signal that we're gone; 1048 * wake it up and exit. We use the driver table lock to insure 1049 * we don't do the wakeup before they're waiting. There is no 1050 * race here because the waiter sleeps on the proc lock for the 1051 * thread so it gets notified at the right time because of an 1052 * extra wakeup that's done in exit1(). 1053 */ 1054 static void 1055 crypto_finis(void *chan) 1056 { 1057 CRYPTO_DRIVER_LOCK(); 1058 wakeup_one(chan); 1059 CRYPTO_DRIVER_UNLOCK(); 1060 kthread_exit(0); 1061 } 1062 1063 /* 1064 * Crypto thread, dispatches crypto requests. 1065 */ 1066 static void 1067 crypto_proc(void) 1068 { 1069 struct cryptop *crp, *submit; 1070 struct cryptkop *krp; 1071 struct cryptocap *cap; 1072 int result, hint; 1073 1074 CRYPTO_Q_LOCK(); 1075 for (;;) { 1076 /* 1077 * Find the first element in the queue that can be 1078 * processed and look-ahead to see if multiple ops 1079 * are ready for the same driver. 1080 */ 1081 submit = NULL; 1082 hint = 0; 1083 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1084 u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid); 1085 cap = crypto_checkdriver(hid); 1086 if (cap == NULL || cap->cc_process == NULL) { 1087 /* Op needs to be migrated, process it. */ 1088 if (submit == NULL) 1089 submit = crp; 1090 break; 1091 } 1092 if (!cap->cc_qblocked) { 1093 if (submit != NULL) { 1094 /* 1095 * We stop on finding another op, 1096 * regardless whether its for the same 1097 * driver or not. We could keep 1098 * searching the queue but it might be 1099 * better to just use a per-driver 1100 * queue instead. 1101 */ 1102 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1103 hint = CRYPTO_HINT_MORE; 1104 break; 1105 } else { 1106 submit = crp; 1107 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1108 break; 1109 /* keep scanning for more are q'd */ 1110 } 1111 } 1112 } 1113 if (submit != NULL) { 1114 TAILQ_REMOVE(&crp_q, submit, crp_next); 1115 result = crypto_invoke(submit, hint); 1116 if (result == ERESTART) { 1117 /* 1118 * The driver ran out of resources, mark the 1119 * driver ``blocked'' for cryptop's and put 1120 * the request back in the queue. It would 1121 * best to put the request back where we got 1122 * it but that's hard so for now we put it 1123 * at the front. This should be ok; putting 1124 * it at the end does not work. 1125 */ 1126 /* XXX validate sid again? */ 1127 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1128 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1129 cryptostats.cs_blocks++; 1130 } 1131 } 1132 1133 /* As above, but for key ops */ 1134 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1135 cap = crypto_checkdriver(krp->krp_hid); 1136 if (cap == NULL || cap->cc_kprocess == NULL) { 1137 /* Op needs to be migrated, process it. */ 1138 break; 1139 } 1140 if (!cap->cc_kqblocked) 1141 break; 1142 } 1143 if (krp != NULL) { 1144 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1145 result = crypto_kinvoke(krp, 0); 1146 if (result == ERESTART) { 1147 /* 1148 * The driver ran out of resources, mark the 1149 * driver ``blocked'' for cryptkop's and put 1150 * the request back in the queue. It would 1151 * best to put the request back where we got 1152 * it but that's hard so for now we put it 1153 * at the front. This should be ok; putting 1154 * it at the end does not work. 1155 */ 1156 /* XXX validate sid again? */ 1157 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1158 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1159 cryptostats.cs_kblocks++; 1160 } 1161 } 1162 1163 if (submit == NULL && krp == NULL) { 1164 /* 1165 * Nothing more to be processed. Sleep until we're 1166 * woken because there are more ops to process. 1167 * This happens either by submission or by a driver 1168 * becoming unblocked and notifying us through 1169 * crypto_unblock. Note that when we wakeup we 1170 * start processing each queue again from the 1171 * front. It's not clear that it's important to 1172 * preserve this ordering since ops may finish 1173 * out of order if dispatched to different devices 1174 * and some become blocked while others do not. 1175 */ 1176 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1177 if (cryptoproc == NULL) 1178 break; 1179 cryptostats.cs_intrs++; 1180 } 1181 } 1182 CRYPTO_Q_UNLOCK(); 1183 1184 crypto_finis(&crp_q); 1185 } 1186 1187 /* 1188 * Crypto returns thread, does callbacks for processed crypto requests. 1189 * Callbacks are done here, rather than in the crypto drivers, because 1190 * callbacks typically are expensive and would slow interrupt handling. 1191 */ 1192 static void 1193 crypto_ret_proc(void) 1194 { 1195 struct cryptop *crpt; 1196 struct cryptkop *krpt; 1197 1198 CRYPTO_RETQ_LOCK(); 1199 for (;;) { 1200 /* Harvest return q's for completed ops */ 1201 crpt = TAILQ_FIRST(&crp_ret_q); 1202 if (crpt != NULL) 1203 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1204 1205 krpt = TAILQ_FIRST(&crp_ret_kq); 1206 if (krpt != NULL) 1207 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1208 1209 if (crpt != NULL || krpt != NULL) { 1210 CRYPTO_RETQ_UNLOCK(); 1211 /* 1212 * Run callbacks unlocked. 1213 */ 1214 if (crpt != NULL) { 1215 #ifdef CRYPTO_TIMING 1216 if (crypto_timing) { 1217 /* 1218 * NB: We must copy the timestamp before 1219 * doing the callback as the cryptop is 1220 * likely to be reclaimed. 1221 */ 1222 struct bintime t = crpt->crp_tstamp; 1223 crypto_tstat(&cryptostats.cs_cb, &t); 1224 crpt->crp_callback(crpt); 1225 crypto_tstat(&cryptostats.cs_finis, &t); 1226 } else 1227 #endif 1228 crpt->crp_callback(crpt); 1229 } 1230 if (krpt != NULL) 1231 krpt->krp_callback(krpt); 1232 CRYPTO_RETQ_LOCK(); 1233 } else { 1234 /* 1235 * Nothing more to be processed. Sleep until we're 1236 * woken because there are more returns to process. 1237 */ 1238 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1239 "crypto_ret_wait", 0); 1240 if (cryptoretproc == NULL) 1241 break; 1242 cryptostats.cs_rets++; 1243 } 1244 } 1245 CRYPTO_RETQ_UNLOCK(); 1246 1247 crypto_finis(&crp_ret_q); 1248 }
Cache object: 40c5800fa886ae0474e21fde4afe6d13
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]