Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/crypto.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $FreeBSD: releng/5.1/sys/opencrypto/crypto.c 111297 2003-02-23 07:25:48Z sam $ */ 2 /* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 3 /* 4 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 5 * 6 * This code was written by Angelos D. Keromytis in Athens, Greece, in 7 * February 2000. Network Security Technologies Inc. (NSTI) kindly 8 * supported the development of this code. 9 * 10 * Copyright (c) 2000, 2001 Angelos D. Keromytis 11 * 12 * Permission to use, copy, and modify this software with or without fee 13 * is hereby granted, provided that this entire notice is included in 14 * all source code copies of any software which is or includes a copy or 15 * modification of this software. 16 * 17 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 18 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 19 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 20 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 21 * PURPOSE. 22 */ 23 #define CRYPTO_TIMING /* enable timing support */ 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/eventhandler.h> 28 #include <sys/kernel.h> 29 #include <sys/kthread.h> 30 #include <sys/lock.h> 31 #include <sys/mutex.h> 32 #include <sys/malloc.h> 33 #include <sys/proc.h> 34 #include <sys/sysctl.h> 35 36 #include <vm/uma.h> 37 #include <opencrypto/cryptodev.h> 38 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 39 40 #define SESID2HID(sid) (((sid) >> 32) & 0xffffffff) 41 42 /* 43 * Crypto drivers register themselves by allocating a slot in the 44 * crypto_drivers table with crypto_get_driverid() and then registering 45 * each algorithm they support with crypto_register() and crypto_kregister(). 46 */ 47 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 48 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 49 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 50 static struct cryptocap *crypto_drivers = NULL; 51 static int crypto_drivers_num = 0; 52 53 /* 54 * There are two queues for crypto requests; one for symmetric (e.g. 55 * cipher) operations and one for asymmetric (e.g. MOD)operations. 56 * A single mutex is used to lock access to both queues. We could 57 * have one per-queue but having one simplifies handling of block/unblock 58 * operations. 59 */ 60 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 61 static TAILQ_HEAD(,cryptkop) crp_kq; 62 static struct mtx crypto_q_mtx; 63 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 64 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 65 66 /* 67 * There are two queues for processing completed crypto requests; one 68 * for the symmetric and one for the asymmetric ops. We only need one 69 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 70 * mutex is used to lock access to both queues. Note that this lock 71 * must be separate from the lock on request queues to insure driver 72 * callbacks don't generate lock order reversals. 73 */ 74 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 75 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 76 static struct mtx crypto_ret_q_mtx; 77 #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 78 #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 79 80 static uma_zone_t cryptop_zone; 81 static uma_zone_t cryptodesc_zone; 82 83 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 84 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 85 &crypto_userasymcrypto, 0, 86 "Enable/disable user-mode access to asymmetric crypto support"); 87 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 88 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 89 &crypto_devallowsoft, 0, 90 "Enable/disable use of software asym crypto support"); 91 92 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 93 94 static void crypto_proc(void); 95 static struct proc *cryptoproc; 96 static void crypto_ret_proc(void); 97 static struct proc *cryptoretproc; 98 static void crypto_destroy(void); 99 static int crypto_invoke(struct cryptop *crp, int hint); 100 static int crypto_kinvoke(struct cryptkop *krp, int hint); 101 102 static struct cryptostats cryptostats; 103 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 104 cryptostats, "Crypto system statistics"); 105 106 #ifdef CRYPTO_TIMING 107 static int crypto_timing = 0; 108 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 109 &crypto_timing, 0, "Enable/disable crypto timing support"); 110 #endif 111 112 static int 113 crypto_init(void) 114 { 115 int error; 116 117 mtx_init(&crypto_drivers_mtx, "crypto driver table", 118 NULL, MTX_DEF|MTX_QUIET); 119 120 TAILQ_INIT(&crp_q); 121 TAILQ_INIT(&crp_kq); 122 mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF); 123 124 TAILQ_INIT(&crp_ret_q); 125 TAILQ_INIT(&crp_ret_kq); 126 mtx_init(&crypto_ret_q_mtx, "crypto return queues", NULL, MTX_DEF); 127 128 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 129 0, 0, 0, 0, 130 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 131 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 132 0, 0, 0, 0, 133 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 134 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 135 printf("crypto_init: cannot setup crypto zones\n"); 136 error = ENOMEM; 137 goto bad; 138 } 139 140 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 141 crypto_drivers = malloc(crypto_drivers_num * 142 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 143 if (crypto_drivers == NULL) { 144 printf("crypto_init: cannot setup crypto drivers\n"); 145 error = ENOMEM; 146 goto bad; 147 } 148 149 error = kthread_create((void (*)(void *)) crypto_proc, NULL, 150 &cryptoproc, 0, 0, "crypto"); 151 if (error) { 152 printf("crypto_init: cannot start crypto thread; error %d", 153 error); 154 goto bad; 155 } 156 157 error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 158 &cryptoretproc, 0, 0, "crypto returns"); 159 if (error) { 160 printf("crypto_init: cannot start cryptoret thread; error %d", 161 error); 162 goto bad; 163 } 164 return 0; 165 bad: 166 crypto_destroy(); 167 return error; 168 } 169 170 /* 171 * Signal a crypto thread to terminate. We use the driver 172 * table lock to synchronize the sleep/wakeups so that we 173 * are sure the threads have terminated before we release 174 * the data structures they use. See crypto_finis below 175 * for the other half of this song-and-dance. 176 */ 177 static void 178 crypto_terminate(struct proc **pp, void *q) 179 { 180 struct proc *p; 181 182 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 183 p = *pp; 184 *pp = NULL; 185 if (p) { 186 wakeup_one(q); 187 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 188 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 189 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 190 PROC_UNLOCK(p); 191 CRYPTO_DRIVER_LOCK(); 192 } 193 } 194 195 static void 196 crypto_destroy(void) 197 { 198 /* 199 * Terminate any crypto threads. 200 */ 201 CRYPTO_DRIVER_LOCK(); 202 crypto_terminate(&cryptoproc, &crp_q); 203 crypto_terminate(&cryptoretproc, &crp_ret_q); 204 CRYPTO_DRIVER_UNLOCK(); 205 206 /* XXX flush queues??? */ 207 208 /* 209 * Reclaim dynamically allocated resources. 210 */ 211 if (crypto_drivers != NULL) 212 free(crypto_drivers, M_CRYPTO_DATA); 213 214 if (cryptodesc_zone != NULL) 215 uma_zdestroy(cryptodesc_zone); 216 if (cryptop_zone != NULL) 217 uma_zdestroy(cryptop_zone); 218 mtx_destroy(&crypto_q_mtx); 219 mtx_destroy(&crypto_ret_q_mtx); 220 mtx_destroy(&crypto_drivers_mtx); 221 } 222 223 /* 224 * Initialization code, both for static and dynamic loading. 225 */ 226 static int 227 crypto_modevent(module_t mod, int type, void *unused) 228 { 229 int error = EINVAL; 230 231 switch (type) { 232 case MOD_LOAD: 233 error = crypto_init(); 234 if (error == 0 && bootverbose) 235 printf("crypto: <crypto core>\n"); 236 break; 237 case MOD_UNLOAD: 238 /*XXX disallow if active sessions */ 239 error = 0; 240 crypto_destroy(); 241 return 0; 242 } 243 return error; 244 } 245 246 static moduledata_t crypto_mod = { 247 "crypto", 248 crypto_modevent, 249 0 250 }; 251 MODULE_VERSION(crypto, 1); 252 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 253 254 /* 255 * Create a new session. 256 */ 257 int 258 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 259 { 260 struct cryptoini *cr; 261 u_int32_t hid, lid; 262 int err = EINVAL; 263 264 CRYPTO_DRIVER_LOCK(); 265 266 if (crypto_drivers == NULL) 267 goto done; 268 269 /* 270 * The algorithm we use here is pretty stupid; just use the 271 * first driver that supports all the algorithms we need. 272 * 273 * XXX We need more smarts here (in real life too, but that's 274 * XXX another story altogether). 275 */ 276 277 for (hid = 0; hid < crypto_drivers_num; hid++) { 278 /* 279 * If it's not initialized or has remaining sessions 280 * referencing it, skip. 281 */ 282 if (crypto_drivers[hid].cc_newsession == NULL || 283 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)) 284 continue; 285 286 /* Hardware required -- ignore software drivers. */ 287 if (hard > 0 && 288 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)) 289 continue; 290 /* Software required -- ignore hardware drivers. */ 291 if (hard < 0 && 292 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 293 continue; 294 295 /* See if all the algorithms are supported. */ 296 for (cr = cri; cr; cr = cr->cri_next) 297 if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) 298 break; 299 300 if (cr == NULL) { 301 /* Ok, all algorithms are supported. */ 302 303 /* 304 * Can't do everything in one session. 305 * 306 * XXX Fix this. We need to inject a "virtual" session layer right 307 * XXX about here. 308 */ 309 310 /* Call the driver initialization routine. */ 311 lid = hid; /* Pass the driver ID. */ 312 err = crypto_drivers[hid].cc_newsession( 313 crypto_drivers[hid].cc_arg, &lid, cri); 314 if (err == 0) { 315 (*sid) = hid; 316 (*sid) <<= 32; 317 (*sid) |= (lid & 0xffffffff); 318 crypto_drivers[hid].cc_sessions++; 319 } 320 break; 321 } 322 } 323 done: 324 CRYPTO_DRIVER_UNLOCK(); 325 return err; 326 } 327 328 /* 329 * Delete an existing session (or a reserved session on an unregistered 330 * driver). 331 */ 332 int 333 crypto_freesession(u_int64_t sid) 334 { 335 u_int32_t hid; 336 int err; 337 338 CRYPTO_DRIVER_LOCK(); 339 340 if (crypto_drivers == NULL) { 341 err = EINVAL; 342 goto done; 343 } 344 345 /* Determine two IDs. */ 346 hid = SESID2HID(sid); 347 348 if (hid >= crypto_drivers_num) { 349 err = ENOENT; 350 goto done; 351 } 352 353 if (crypto_drivers[hid].cc_sessions) 354 crypto_drivers[hid].cc_sessions--; 355 356 /* Call the driver cleanup routine, if available. */ 357 if (crypto_drivers[hid].cc_freesession) 358 err = crypto_drivers[hid].cc_freesession( 359 crypto_drivers[hid].cc_arg, sid); 360 else 361 err = 0; 362 363 /* 364 * If this was the last session of a driver marked as invalid, 365 * make the entry available for reuse. 366 */ 367 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 368 crypto_drivers[hid].cc_sessions == 0) 369 bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 370 371 done: 372 CRYPTO_DRIVER_UNLOCK(); 373 return err; 374 } 375 376 /* 377 * Return an unused driver id. Used by drivers prior to registering 378 * support for the algorithms they handle. 379 */ 380 int32_t 381 crypto_get_driverid(u_int32_t flags) 382 { 383 struct cryptocap *newdrv; 384 int i; 385 386 CRYPTO_DRIVER_LOCK(); 387 388 for (i = 0; i < crypto_drivers_num; i++) 389 if (crypto_drivers[i].cc_process == NULL && 390 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 391 crypto_drivers[i].cc_sessions == 0) 392 break; 393 394 /* Out of entries, allocate some more. */ 395 if (i == crypto_drivers_num) { 396 /* Be careful about wrap-around. */ 397 if (2 * crypto_drivers_num <= crypto_drivers_num) { 398 CRYPTO_DRIVER_UNLOCK(); 399 printf("crypto: driver count wraparound!\n"); 400 return -1; 401 } 402 403 newdrv = malloc(2 * crypto_drivers_num * 404 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 405 if (newdrv == NULL) { 406 CRYPTO_DRIVER_UNLOCK(); 407 printf("crypto: no space to expand driver table!\n"); 408 return -1; 409 } 410 411 bcopy(crypto_drivers, newdrv, 412 crypto_drivers_num * sizeof(struct cryptocap)); 413 414 crypto_drivers_num *= 2; 415 416 free(crypto_drivers, M_CRYPTO_DATA); 417 crypto_drivers = newdrv; 418 } 419 420 /* NB: state is zero'd on free */ 421 crypto_drivers[i].cc_sessions = 1; /* Mark */ 422 crypto_drivers[i].cc_flags = flags; 423 if (bootverbose) 424 printf("crypto: assign driver %u, flags %u\n", i, flags); 425 426 CRYPTO_DRIVER_UNLOCK(); 427 428 return i; 429 } 430 431 static struct cryptocap * 432 crypto_checkdriver(u_int32_t hid) 433 { 434 if (crypto_drivers == NULL) 435 return NULL; 436 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 437 } 438 439 /* 440 * Register support for a key-related algorithm. This routine 441 * is called once for each algorithm supported a driver. 442 */ 443 int 444 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 445 int (*kprocess)(void*, struct cryptkop *, int), 446 void *karg) 447 { 448 struct cryptocap *cap; 449 int err; 450 451 CRYPTO_DRIVER_LOCK(); 452 453 cap = crypto_checkdriver(driverid); 454 if (cap != NULL && 455 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 456 /* 457 * XXX Do some performance testing to determine placing. 458 * XXX We probably need an auxiliary data structure that 459 * XXX describes relative performances. 460 */ 461 462 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 463 if (bootverbose) 464 printf("crypto: driver %u registers key alg %u flags %u\n" 465 , driverid 466 , kalg 467 , flags 468 ); 469 470 if (cap->cc_kprocess == NULL) { 471 cap->cc_karg = karg; 472 cap->cc_kprocess = kprocess; 473 } 474 err = 0; 475 } else 476 err = EINVAL; 477 478 CRYPTO_DRIVER_UNLOCK(); 479 return err; 480 } 481 482 /* 483 * Register support for a non-key-related algorithm. This routine 484 * is called once for each such algorithm supported by a driver. 485 */ 486 int 487 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 488 u_int32_t flags, 489 int (*newses)(void*, u_int32_t*, struct cryptoini*), 490 int (*freeses)(void*, u_int64_t), 491 int (*process)(void*, struct cryptop *, int), 492 void *arg) 493 { 494 struct cryptocap *cap; 495 int err; 496 497 CRYPTO_DRIVER_LOCK(); 498 499 cap = crypto_checkdriver(driverid); 500 /* NB: algorithms are in the range [1..max] */ 501 if (cap != NULL && 502 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 503 /* 504 * XXX Do some performance testing to determine placing. 505 * XXX We probably need an auxiliary data structure that 506 * XXX describes relative performances. 507 */ 508 509 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 510 cap->cc_max_op_len[alg] = maxoplen; 511 if (bootverbose) 512 printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 513 , driverid 514 , alg 515 , flags 516 , maxoplen 517 ); 518 519 if (cap->cc_process == NULL) { 520 cap->cc_arg = arg; 521 cap->cc_newsession = newses; 522 cap->cc_process = process; 523 cap->cc_freesession = freeses; 524 cap->cc_sessions = 0; /* Unmark */ 525 } 526 err = 0; 527 } else 528 err = EINVAL; 529 530 CRYPTO_DRIVER_UNLOCK(); 531 return err; 532 } 533 534 /* 535 * Unregister a crypto driver. If there are pending sessions using it, 536 * leave enough information around so that subsequent calls using those 537 * sessions will correctly detect the driver has been unregistered and 538 * reroute requests. 539 */ 540 int 541 crypto_unregister(u_int32_t driverid, int alg) 542 { 543 int i, err; 544 u_int32_t ses; 545 struct cryptocap *cap; 546 547 CRYPTO_DRIVER_LOCK(); 548 549 cap = crypto_checkdriver(driverid); 550 if (cap != NULL && 551 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 552 cap->cc_alg[alg] != 0) { 553 cap->cc_alg[alg] = 0; 554 cap->cc_max_op_len[alg] = 0; 555 556 /* Was this the last algorithm ? */ 557 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 558 if (cap->cc_alg[i] != 0) 559 break; 560 561 if (i == CRYPTO_ALGORITHM_MAX + 1) { 562 ses = cap->cc_sessions; 563 bzero(cap, sizeof(struct cryptocap)); 564 if (ses != 0) { 565 /* 566 * If there are pending sessions, just mark as invalid. 567 */ 568 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 569 cap->cc_sessions = ses; 570 } 571 } 572 err = 0; 573 } else 574 err = EINVAL; 575 576 CRYPTO_DRIVER_UNLOCK(); 577 return err; 578 } 579 580 /* 581 * Unregister all algorithms associated with a crypto driver. 582 * If there are pending sessions using it, leave enough information 583 * around so that subsequent calls using those sessions will 584 * correctly detect the driver has been unregistered and reroute 585 * requests. 586 */ 587 int 588 crypto_unregister_all(u_int32_t driverid) 589 { 590 int i, err; 591 u_int32_t ses; 592 struct cryptocap *cap; 593 594 CRYPTO_DRIVER_LOCK(); 595 596 cap = crypto_checkdriver(driverid); 597 if (cap != NULL) { 598 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 599 cap->cc_alg[i] = 0; 600 cap->cc_max_op_len[i] = 0; 601 } 602 ses = cap->cc_sessions; 603 bzero(cap, sizeof(struct cryptocap)); 604 if (ses != 0) { 605 /* 606 * If there are pending sessions, just mark as invalid. 607 */ 608 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 609 cap->cc_sessions = ses; 610 } 611 err = 0; 612 } else 613 err = EINVAL; 614 615 CRYPTO_DRIVER_UNLOCK(); 616 return err; 617 } 618 619 /* 620 * Clear blockage on a driver. The what parameter indicates whether 621 * the driver is now ready for cryptop's and/or cryptokop's. 622 */ 623 int 624 crypto_unblock(u_int32_t driverid, int what) 625 { 626 struct cryptocap *cap; 627 int needwakeup, err; 628 629 CRYPTO_Q_LOCK(); 630 cap = crypto_checkdriver(driverid); 631 if (cap != NULL) { 632 needwakeup = 0; 633 if (what & CRYPTO_SYMQ) { 634 needwakeup |= cap->cc_qblocked; 635 cap->cc_qblocked = 0; 636 } 637 if (what & CRYPTO_ASYMQ) { 638 needwakeup |= cap->cc_kqblocked; 639 cap->cc_kqblocked = 0; 640 } 641 if (needwakeup) 642 wakeup_one(&crp_q); 643 err = 0; 644 } else 645 err = EINVAL; 646 CRYPTO_Q_UNLOCK(); 647 648 return err; 649 } 650 651 /* 652 * Add a crypto request to a queue, to be processed by the kernel thread. 653 */ 654 int 655 crypto_dispatch(struct cryptop *crp) 656 { 657 u_int32_t hid = SESID2HID(crp->crp_sid); 658 int result; 659 660 cryptostats.cs_ops++; 661 662 #ifdef CRYPTO_TIMING 663 if (crypto_timing) 664 binuptime(&crp->crp_tstamp); 665 #endif 666 667 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 668 struct cryptocap *cap; 669 /* 670 * Caller marked the request to be processed 671 * immediately; dispatch it directly to the 672 * driver unless the driver is currently blocked. 673 */ 674 cap = crypto_checkdriver(hid); 675 if (cap && !cap->cc_qblocked) { 676 result = crypto_invoke(crp, 0); 677 if (result == ERESTART) { 678 /* 679 * The driver ran out of resources, mark the 680 * driver ``blocked'' for cryptop's and put 681 * the request on the queue. 682 */ 683 CRYPTO_Q_LOCK(); 684 crypto_drivers[hid].cc_qblocked = 1; 685 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); 686 CRYPTO_Q_UNLOCK(); 687 cryptostats.cs_blocks++; 688 } 689 } else { 690 /* 691 * The driver is blocked, just queue the op until 692 * it unblocks and the kernel thread gets kicked. 693 */ 694 CRYPTO_Q_LOCK(); 695 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 696 CRYPTO_Q_UNLOCK(); 697 result = 0; 698 } 699 } else { 700 int wasempty; 701 /* 702 * Caller marked the request as ``ok to delay''; 703 * queue it for the dispatch thread. This is desirable 704 * when the operation is low priority and/or suitable 705 * for batching. 706 */ 707 CRYPTO_Q_LOCK(); 708 wasempty = TAILQ_EMPTY(&crp_q); 709 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 710 if (wasempty) 711 wakeup_one(&crp_q); 712 CRYPTO_Q_UNLOCK(); 713 result = 0; 714 } 715 716 return result; 717 } 718 719 /* 720 * Add an asymetric crypto request to a queue, 721 * to be processed by the kernel thread. 722 */ 723 int 724 crypto_kdispatch(struct cryptkop *krp) 725 { 726 struct cryptocap *cap; 727 int result; 728 729 cryptostats.cs_kops++; 730 731 CRYPTO_Q_LOCK(); 732 cap = crypto_checkdriver(krp->krp_hid); 733 if (cap && !cap->cc_kqblocked) { 734 result = crypto_kinvoke(krp, 0); 735 if (result == ERESTART) { 736 /* 737 * The driver ran out of resources, mark the 738 * driver ``blocked'' for cryptkop's and put 739 * the request back in the queue. It would 740 * best to put the request back where we got 741 * it but that's hard so for now we put it 742 * at the front. This should be ok; putting 743 * it at the end does not work. 744 */ 745 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 746 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 747 cryptostats.cs_kblocks++; 748 } 749 } else { 750 /* 751 * The driver is blocked, just queue the op until 752 * it unblocks and the kernel thread gets kicked. 753 */ 754 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 755 result = 0; 756 } 757 CRYPTO_Q_UNLOCK(); 758 759 return result; 760 } 761 762 /* 763 * Dispatch an assymetric crypto request to the appropriate crypto devices. 764 */ 765 static int 766 crypto_kinvoke(struct cryptkop *krp, int hint) 767 { 768 u_int32_t hid; 769 int error; 770 771 mtx_assert(&crypto_q_mtx, MA_OWNED); 772 773 /* Sanity checks. */ 774 if (krp == NULL) 775 return EINVAL; 776 if (krp->krp_callback == NULL) { 777 free(krp, M_XDATA); /* XXX allocated in cryptodev */ 778 return EINVAL; 779 } 780 781 for (hid = 0; hid < crypto_drivers_num; hid++) { 782 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 783 !crypto_devallowsoft) 784 continue; 785 if (crypto_drivers[hid].cc_kprocess == NULL) 786 continue; 787 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 788 CRYPTO_ALG_FLAG_SUPPORTED) == 0) 789 continue; 790 break; 791 } 792 if (hid < crypto_drivers_num) { 793 krp->krp_hid = hid; 794 error = crypto_drivers[hid].cc_kprocess( 795 crypto_drivers[hid].cc_karg, krp, hint); 796 } else 797 error = ENODEV; 798 799 if (error) { 800 krp->krp_status = error; 801 crypto_kdone(krp); 802 } 803 return 0; 804 } 805 806 #ifdef CRYPTO_TIMING 807 static void 808 crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 809 { 810 struct bintime now, delta; 811 struct timespec t; 812 uint64_t u; 813 814 binuptime(&now); 815 u = now.frac; 816 delta.frac = now.frac - bt->frac; 817 delta.sec = now.sec - bt->sec; 818 if (u < delta.frac) 819 delta.sec--; 820 bintime2timespec(&delta, &t); 821 timespecadd(&ts->acc, &t); 822 if (timespeccmp(&t, &ts->min, <)) 823 ts->min = t; 824 if (timespeccmp(&t, &ts->max, >)) 825 ts->max = t; 826 ts->count++; 827 828 *bt = now; 829 } 830 #endif 831 832 /* 833 * Dispatch a crypto request to the appropriate crypto devices. 834 */ 835 static int 836 crypto_invoke(struct cryptop *crp, int hint) 837 { 838 u_int32_t hid; 839 int (*process)(void*, struct cryptop *, int); 840 841 #ifdef CRYPTO_TIMING 842 if (crypto_timing) 843 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 844 #endif 845 /* Sanity checks. */ 846 if (crp == NULL) 847 return EINVAL; 848 if (crp->crp_callback == NULL) { 849 crypto_freereq(crp); 850 return EINVAL; 851 } 852 if (crp->crp_desc == NULL) { 853 crp->crp_etype = EINVAL; 854 crypto_done(crp); 855 return 0; 856 } 857 858 hid = SESID2HID(crp->crp_sid); 859 if (hid < crypto_drivers_num) { 860 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 861 crypto_freesession(crp->crp_sid); 862 process = crypto_drivers[hid].cc_process; 863 } else { 864 process = NULL; 865 } 866 867 if (process == NULL) { 868 struct cryptodesc *crd; 869 u_int64_t nid; 870 871 /* 872 * Driver has unregistered; migrate the session and return 873 * an error to the caller so they'll resubmit the op. 874 */ 875 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 876 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 877 878 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 879 crp->crp_sid = nid; 880 881 crp->crp_etype = EAGAIN; 882 crypto_done(crp); 883 return 0; 884 } else { 885 /* 886 * Invoke the driver to process the request. 887 */ 888 return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 889 } 890 } 891 892 /* 893 * Release a set of crypto descriptors. 894 */ 895 void 896 crypto_freereq(struct cryptop *crp) 897 { 898 struct cryptodesc *crd; 899 900 if (crp == NULL) 901 return; 902 903 while ((crd = crp->crp_desc) != NULL) { 904 crp->crp_desc = crd->crd_next; 905 uma_zfree(cryptodesc_zone, crd); 906 } 907 908 uma_zfree(cryptop_zone, crp); 909 } 910 911 /* 912 * Acquire a set of crypto descriptors. 913 */ 914 struct cryptop * 915 crypto_getreq(int num) 916 { 917 struct cryptodesc *crd; 918 struct cryptop *crp; 919 920 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 921 if (crp != NULL) { 922 while (num--) { 923 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 924 if (crd == NULL) { 925 crypto_freereq(crp); 926 return NULL; 927 } 928 929 crd->crd_next = crp->crp_desc; 930 crp->crp_desc = crd; 931 } 932 } 933 return crp; 934 } 935 936 /* 937 * Invoke the callback on behalf of the driver. 938 */ 939 void 940 crypto_done(struct cryptop *crp) 941 { 942 if (crp->crp_etype != 0) 943 cryptostats.cs_errs++; 944 #ifdef CRYPTO_TIMING 945 if (crypto_timing) 946 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 947 #endif 948 if (crp->crp_flags & CRYPTO_F_CBIMM) { 949 /* 950 * Do the callback directly. This is ok when the 951 * callback routine does very little (e.g. the 952 * /dev/crypto callback method just does a wakeup). 953 */ 954 #ifdef CRYPTO_TIMING 955 if (crypto_timing) { 956 /* 957 * NB: We must copy the timestamp before 958 * doing the callback as the cryptop is 959 * likely to be reclaimed. 960 */ 961 struct bintime t = crp->crp_tstamp; 962 crypto_tstat(&cryptostats.cs_cb, &t); 963 crp->crp_callback(crp); 964 crypto_tstat(&cryptostats.cs_finis, &t); 965 } else 966 #endif 967 crp->crp_callback(crp); 968 } else { 969 int wasempty; 970 /* 971 * Normal case; queue the callback for the thread. 972 */ 973 CRYPTO_RETQ_LOCK(); 974 wasempty = TAILQ_EMPTY(&crp_ret_q); 975 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 976 977 if (wasempty) 978 wakeup_one(&crp_ret_q); /* shared wait channel */ 979 CRYPTO_RETQ_UNLOCK(); 980 } 981 } 982 983 /* 984 * Invoke the callback on behalf of the driver. 985 */ 986 void 987 crypto_kdone(struct cryptkop *krp) 988 { 989 int wasempty; 990 991 if (krp->krp_status != 0) 992 cryptostats.cs_kerrs++; 993 CRYPTO_RETQ_LOCK(); 994 wasempty = TAILQ_EMPTY(&crp_ret_kq); 995 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 996 997 if (wasempty) 998 wakeup_one(&crp_ret_q); /* shared wait channel */ 999 CRYPTO_RETQ_UNLOCK(); 1000 } 1001 1002 int 1003 crypto_getfeat(int *featp) 1004 { 1005 int hid, kalg, feat = 0; 1006 1007 if (!crypto_userasymcrypto) 1008 goto out; 1009 1010 CRYPTO_DRIVER_LOCK(); 1011 for (hid = 0; hid < crypto_drivers_num; hid++) { 1012 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 1013 !crypto_devallowsoft) { 1014 continue; 1015 } 1016 if (crypto_drivers[hid].cc_kprocess == NULL) 1017 continue; 1018 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1019 if ((crypto_drivers[hid].cc_kalg[kalg] & 1020 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1021 feat |= 1 << kalg; 1022 } 1023 CRYPTO_DRIVER_UNLOCK(); 1024 out: 1025 *featp = feat; 1026 return (0); 1027 } 1028 1029 /* 1030 * Terminate a thread at module unload. The process that 1031 * initiated this is waiting for us to signal that we're gone; 1032 * wake it up and exit. We use the driver table lock to insure 1033 * we don't do the wakeup before they're waiting. There is no 1034 * race here because the waiter sleeps on the proc lock for the 1035 * thread so it gets notified at the right time because of an 1036 * extra wakeup that's done in exit1(). 1037 */ 1038 static void 1039 crypto_finis(void *chan) 1040 { 1041 CRYPTO_DRIVER_LOCK(); 1042 wakeup_one(chan); 1043 CRYPTO_DRIVER_UNLOCK(); 1044 mtx_lock(&Giant); 1045 kthread_exit(0); 1046 } 1047 1048 /* 1049 * Crypto thread, dispatches crypto requests. 1050 */ 1051 static void 1052 crypto_proc(void) 1053 { 1054 struct cryptop *crp, *submit; 1055 struct cryptkop *krp; 1056 struct cryptocap *cap; 1057 int result, hint; 1058 1059 CRYPTO_Q_LOCK(); 1060 for (;;) { 1061 /* 1062 * Find the first element in the queue that can be 1063 * processed and look-ahead to see if multiple ops 1064 * are ready for the same driver. 1065 */ 1066 submit = NULL; 1067 hint = 0; 1068 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1069 u_int32_t hid = SESID2HID(crp->crp_sid); 1070 cap = crypto_checkdriver(hid); 1071 if (cap == NULL || cap->cc_process == NULL) { 1072 /* Op needs to be migrated, process it. */ 1073 if (submit == NULL) 1074 submit = crp; 1075 break; 1076 } 1077 if (!cap->cc_qblocked) { 1078 if (submit != NULL) { 1079 /* 1080 * We stop on finding another op, 1081 * regardless whether its for the same 1082 * driver or not. We could keep 1083 * searching the queue but it might be 1084 * better to just use a per-driver 1085 * queue instead. 1086 */ 1087 if (SESID2HID(submit->crp_sid) == hid) 1088 hint = CRYPTO_HINT_MORE; 1089 break; 1090 } else { 1091 submit = crp; 1092 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1093 break; 1094 /* keep scanning for more are q'd */ 1095 } 1096 } 1097 } 1098 if (submit != NULL) { 1099 TAILQ_REMOVE(&crp_q, submit, crp_next); 1100 result = crypto_invoke(submit, hint); 1101 if (result == ERESTART) { 1102 /* 1103 * The driver ran out of resources, mark the 1104 * driver ``blocked'' for cryptop's and put 1105 * the request back in the queue. It would 1106 * best to put the request back where we got 1107 * it but that's hard so for now we put it 1108 * at the front. This should be ok; putting 1109 * it at the end does not work. 1110 */ 1111 /* XXX validate sid again? */ 1112 crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1113 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1114 cryptostats.cs_blocks++; 1115 } 1116 } 1117 1118 /* As above, but for key ops */ 1119 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1120 cap = crypto_checkdriver(krp->krp_hid); 1121 if (cap == NULL || cap->cc_kprocess == NULL) { 1122 /* Op needs to be migrated, process it. */ 1123 break; 1124 } 1125 if (!cap->cc_kqblocked) 1126 break; 1127 } 1128 if (krp != NULL) { 1129 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1130 result = crypto_kinvoke(krp, 0); 1131 if (result == ERESTART) { 1132 /* 1133 * The driver ran out of resources, mark the 1134 * driver ``blocked'' for cryptkop's and put 1135 * the request back in the queue. It would 1136 * best to put the request back where we got 1137 * it but that's hard so for now we put it 1138 * at the front. This should be ok; putting 1139 * it at the end does not work. 1140 */ 1141 /* XXX validate sid again? */ 1142 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1143 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1144 cryptostats.cs_kblocks++; 1145 } 1146 } 1147 1148 if (submit == NULL && krp == NULL) { 1149 /* 1150 * Nothing more to be processed. Sleep until we're 1151 * woken because there are more ops to process. 1152 * This happens either by submission or by a driver 1153 * becoming unblocked and notifying us through 1154 * crypto_unblock. Note that when we wakeup we 1155 * start processing each queue again from the 1156 * front. It's not clear that it's important to 1157 * preserve this ordering since ops may finish 1158 * out of order if dispatched to different devices 1159 * and some become blocked while others do not. 1160 */ 1161 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1162 if (cryptoproc == NULL) 1163 break; 1164 cryptostats.cs_intrs++; 1165 } 1166 } 1167 CRYPTO_Q_UNLOCK(); 1168 1169 crypto_finis(&crp_q); 1170 } 1171 1172 /* 1173 * Crypto returns thread, does callbacks for processed crypto requests. 1174 * Callbacks are done here, rather than in the crypto drivers, because 1175 * callbacks typically are expensive and would slow interrupt handling. 1176 */ 1177 static void 1178 crypto_ret_proc(void) 1179 { 1180 struct cryptop *crpt; 1181 struct cryptkop *krpt; 1182 1183 CRYPTO_RETQ_LOCK(); 1184 for (;;) { 1185 /* Harvest return q's for completed ops */ 1186 crpt = TAILQ_FIRST(&crp_ret_q); 1187 if (crpt != NULL) 1188 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1189 1190 krpt = TAILQ_FIRST(&crp_ret_kq); 1191 if (krpt != NULL) 1192 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1193 1194 if (crpt != NULL || krpt != NULL) { 1195 CRYPTO_RETQ_UNLOCK(); 1196 /* 1197 * Run callbacks unlocked. 1198 */ 1199 if (crpt != NULL) { 1200 #ifdef CRYPTO_TIMING 1201 if (crypto_timing) { 1202 /* 1203 * NB: We must copy the timestamp before 1204 * doing the callback as the cryptop is 1205 * likely to be reclaimed. 1206 */ 1207 struct bintime t = crpt->crp_tstamp; 1208 crypto_tstat(&cryptostats.cs_cb, &t); 1209 crpt->crp_callback(crpt); 1210 crypto_tstat(&cryptostats.cs_finis, &t); 1211 } else 1212 #endif 1213 crpt->crp_callback(crpt); 1214 } 1215 if (krpt != NULL) 1216 krpt->krp_callback(krpt); 1217 CRYPTO_RETQ_LOCK(); 1218 } else { 1219 /* 1220 * Nothing more to be processed. Sleep until we're 1221 * woken because there are more returns to process. 1222 */ 1223 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1224 "crypto_ret_wait", 0); 1225 if (cryptoretproc == NULL) 1226 break; 1227 cryptostats.cs_rets++; 1228 } 1229 } 1230 CRYPTO_RETQ_UNLOCK(); 1231 1232 crypto_finis(&crp_ret_q); 1233 }
Cache object: 291e07a3d133271bc98afd7104245c32
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]