ccp.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2017 Chelsio Communications, Inc.
    5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
    6  * All rights reserved.
    7  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include "opt_ddb.h"
   35 
   36 #include <sys/param.h>
   37 #include <sys/bus.h>
   38 #include <sys/lock.h>
   39 #include <sys/kernel.h>
   40 #include <sys/malloc.h>
   41 #include <sys/mutex.h>
   42 #include <sys/module.h>
   43 #include <sys/random.h>
   44 #include <sys/sglist.h>
   45 #include <sys/sysctl.h>
   46 
   47 #ifdef DDB
   48 #include <ddb/ddb.h>
   49 #endif
   50 
   51 #include <dev/pci/pcivar.h>
   52 
   53 #include <dev/random/randomdev.h>
   54 
   55 #include <opencrypto/cryptodev.h>
   56 #include <opencrypto/xform.h>
   57 
   58 #include "cryptodev_if.h"
   59 
   60 #include "ccp.h"
   61 #include "ccp_hardware.h"
   62 
   63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
   64 
   65 /*
   66  * Need a global softc available for garbage random_source API, which lacks any
   67  * context pointer.  It's also handy for debugging.
   68  */
   69 struct ccp_softc *g_ccp_softc;
   70 
   71 bool g_debug_print = false;
   72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
   73     "Set to enable debugging log messages");
   74 
   75 static struct pciid {
   76         uint32_t devid;
   77         const char *desc;
   78 } ccp_ids[] = {
   79         { 0x14561022, "AMD CCP-5a" },
   80         { 0x14681022, "AMD CCP-5b" },
   81         { 0x15df1022, "AMD CCP-5a" },
   82 };
   83 
   84 static struct random_source random_ccp = {
   85         .rs_ident = "AMD CCP TRNG",
   86         .rs_source = RANDOM_PURE_CCP,
   87         .rs_read = random_ccp_read,
   88 };
   89 
   90 /*
   91  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
   92  * crypto operation buffer.
   93  */
   94 static int
   95 ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
   96 {
   97         int error;
   98 
   99         sglist_reset(sg);
  100         switch (cb->cb_type) {
  101         case CRYPTO_BUF_MBUF:
  102                 error = sglist_append_mbuf(sg, cb->cb_mbuf);
  103                 break;
  104         case CRYPTO_BUF_SINGLE_MBUF:
  105                 error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
  106                 break;
  107         case CRYPTO_BUF_UIO:
  108                 error = sglist_append_uio(sg, cb->cb_uio);
  109                 break;
  110         case CRYPTO_BUF_CONTIG:
  111                 error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
  112                 break;
  113         case CRYPTO_BUF_VMPAGE:
  114                 error = sglist_append_vmpages(sg, cb->cb_vm_page,
  115                     cb->cb_vm_page_len, cb->cb_vm_page_offset);
  116                 break;
  117         default:
  118                 error = EINVAL;
  119         }
  120         return (error);
  121 }
  122 
  123 static int
  124 ccp_probe(device_t dev)
  125 {
  126         struct pciid *ip;
  127         uint32_t id;
  128 
  129         id = pci_get_devid(dev);
  130         for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
  131                 if (id == ip->devid) {
  132                         device_set_desc(dev, ip->desc);
  133                         return (0);
  134                 }
  135         }
  136         return (ENXIO);
  137 }
  138 
  139 static void
  140 ccp_initialize_queues(struct ccp_softc *sc)
  141 {
  142         struct ccp_queue *qp;
  143         size_t i;
  144 
  145         for (i = 0; i < nitems(sc->queues); i++) {
  146                 qp = &sc->queues[i];
  147 
  148                 qp->cq_softc = sc;
  149                 qp->cq_qindex = i;
  150                 mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
  151                 /* XXX - arbitrarily chosen sizes */
  152                 qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
  153                 /* Two more SGEs than sg_crp to accommodate ipad. */
  154                 qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
  155                 qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
  156         }
  157 }
  158 
  159 static void
  160 ccp_free_queues(struct ccp_softc *sc)
  161 {
  162         struct ccp_queue *qp;
  163         size_t i;
  164 
  165         for (i = 0; i < nitems(sc->queues); i++) {
  166                 qp = &sc->queues[i];
  167 
  168                 mtx_destroy(&qp->cq_lock);
  169                 sglist_free(qp->cq_sg_crp);
  170                 sglist_free(qp->cq_sg_ulptx);
  171                 sglist_free(qp->cq_sg_dst);
  172         }
  173 }
  174 
  175 static int
  176 ccp_attach(device_t dev)
  177 {
  178         struct ccp_softc *sc;
  179         int error;
  180 
  181         sc = device_get_softc(dev);
  182         sc->dev = dev;
  183 
  184         sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
  185             CRYPTOCAP_F_HARDWARE);
  186         if (sc->cid < 0) {
  187                 device_printf(dev, "could not get crypto driver id\n");
  188                 return (ENXIO);
  189         }
  190 
  191         error = ccp_hw_attach(dev);
  192         if (error != 0)
  193                 return (error);
  194 
  195         mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
  196 
  197         ccp_initialize_queues(sc);
  198 
  199         if (g_ccp_softc == NULL) {
  200                 g_ccp_softc = sc;
  201                 if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
  202                         random_source_register(&random_ccp);
  203         }
  204 
  205         return (0);
  206 }
  207 
  208 static int
  209 ccp_detach(device_t dev)
  210 {
  211         struct ccp_softc *sc;
  212 
  213         sc = device_get_softc(dev);
  214 
  215         mtx_lock(&sc->lock);
  216         sc->detaching = true;
  217         mtx_unlock(&sc->lock);
  218 
  219         crypto_unregister_all(sc->cid);
  220         if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
  221                 random_source_deregister(&random_ccp);
  222 
  223         ccp_hw_detach(dev);
  224         ccp_free_queues(sc);
  225 
  226         if (g_ccp_softc == sc)
  227                 g_ccp_softc = NULL;
  228 
  229         mtx_destroy(&sc->lock);
  230         return (0);
  231 }
  232 
  233 static void
  234 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
  235 {
  236         union authctx auth_ctx;
  237         const struct auth_hash *axf;
  238         u_int i;
  239 
  240         /*
  241          * If the key is larger than the block size, use the digest of
  242          * the key as the key instead.
  243          */
  244         axf = s->hmac.auth_hash;
  245         if (klen > axf->blocksize) {
  246                 axf->Init(&auth_ctx);
  247                 axf->Update(&auth_ctx, key, klen);
  248                 axf->Final(s->hmac.ipad, &auth_ctx);
  249                 explicit_bzero(&auth_ctx, sizeof(auth_ctx));
  250                 klen = axf->hashsize;
  251         } else
  252                 memcpy(s->hmac.ipad, key, klen);
  253 
  254         memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
  255         memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
  256 
  257         for (i = 0; i < axf->blocksize; i++) {
  258                 s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
  259                 s->hmac.opad[i] ^= HMAC_OPAD_VAL;
  260         }
  261 }
  262 
  263 static bool
  264 ccp_aes_check_keylen(int alg, int klen)
  265 {
  266 
  267         switch (klen * 8) {
  268         case 128:
  269         case 192:
  270                 if (alg == CRYPTO_AES_XTS)
  271                         return (false);
  272                 break;
  273         case 256:
  274                 break;
  275         case 512:
  276                 if (alg != CRYPTO_AES_XTS)
  277                         return (false);
  278                 break;
  279         default:
  280                 return (false);
  281         }
  282         return (true);
  283 }
  284 
  285 static void
  286 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
  287 {
  288         unsigned kbits;
  289 
  290         if (alg == CRYPTO_AES_XTS)
  291                 kbits = (klen / 2) * 8;
  292         else
  293                 kbits = klen * 8;
  294 
  295         switch (kbits) {
  296         case 128:
  297                 s->blkcipher.cipher_type = CCP_AES_TYPE_128;
  298                 break;
  299         case 192:
  300                 s->blkcipher.cipher_type = CCP_AES_TYPE_192;
  301                 break;
  302         case 256:
  303                 s->blkcipher.cipher_type = CCP_AES_TYPE_256;
  304                 break;
  305         default:
  306                 panic("should not get here");
  307         }
  308 
  309         s->blkcipher.key_len = klen;
  310         memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
  311 }
  312 
  313 static bool
  314 ccp_auth_supported(struct ccp_softc *sc,
  315     const struct crypto_session_params *csp)
  316 {
  317         
  318         if ((sc->hw_features & VERSION_CAP_SHA) == 0)
  319                 return (false);
  320         switch (csp->csp_auth_alg) {
  321         case CRYPTO_SHA1_HMAC:
  322         case CRYPTO_SHA2_256_HMAC:
  323         case CRYPTO_SHA2_384_HMAC:
  324         case CRYPTO_SHA2_512_HMAC:
  325                 if (csp->csp_auth_key == NULL)
  326                         return (false);
  327                 break;
  328         default:
  329                 return (false);
  330         }
  331         return (true);
  332 }
  333 
  334 static bool
  335 ccp_cipher_supported(struct ccp_softc *sc,
  336     const struct crypto_session_params *csp)
  337 {
  338 
  339         if ((sc->hw_features & VERSION_CAP_AES) == 0)
  340                 return (false);
  341         switch (csp->csp_cipher_alg) {
  342         case CRYPTO_AES_CBC:
  343                 if (csp->csp_ivlen != AES_BLOCK_LEN)
  344                         return (false);
  345                 break;
  346         case CRYPTO_AES_ICM:
  347                 if (csp->csp_ivlen != AES_BLOCK_LEN)
  348                         return (false);
  349                 break;
  350         case CRYPTO_AES_XTS:
  351                 if (csp->csp_ivlen != AES_XTS_IV_LEN)
  352                         return (false);
  353                 break;
  354         default:
  355                 return (false);
  356         }
  357         return (ccp_aes_check_keylen(csp->csp_cipher_alg,
  358             csp->csp_cipher_klen));
  359 }
  360 
  361 static int
  362 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
  363 {
  364         struct ccp_softc *sc;
  365 
  366         if (csp->csp_flags != 0)
  367                 return (EINVAL);
  368         sc = device_get_softc(dev);
  369         switch (csp->csp_mode) {
  370         case CSP_MODE_DIGEST:
  371                 if (!ccp_auth_supported(sc, csp))
  372                         return (EINVAL);
  373                 break;
  374         case CSP_MODE_CIPHER:
  375                 if (!ccp_cipher_supported(sc, csp))
  376                         return (EINVAL);
  377                 break;
  378         case CSP_MODE_AEAD:
  379                 switch (csp->csp_cipher_alg) {
  380                 case CRYPTO_AES_NIST_GCM_16:
  381                         if ((sc->hw_features & VERSION_CAP_AES) == 0)
  382                                 return (EINVAL);
  383                         break;
  384                 default:
  385                         return (EINVAL);
  386                 }
  387                 break;
  388         case CSP_MODE_ETA:
  389                 if (!ccp_auth_supported(sc, csp) ||
  390                     !ccp_cipher_supported(sc, csp))
  391                         return (EINVAL);
  392                 break;
  393         default:
  394                 return (EINVAL);
  395         }
  396 
  397         return (CRYPTODEV_PROBE_HARDWARE);
  398 }
  399 
  400 static int
  401 ccp_newsession(device_t dev, crypto_session_t cses,
  402     const struct crypto_session_params *csp)
  403 {
  404         struct ccp_softc *sc;
  405         struct ccp_session *s;
  406         const struct auth_hash *auth_hash;
  407         enum ccp_aes_mode cipher_mode;
  408         unsigned auth_mode;
  409         unsigned q;
  410 
  411         /* XXX reconcile auth_mode with use by ccp_sha */
  412         switch (csp->csp_auth_alg) {
  413         case CRYPTO_SHA1_HMAC:
  414                 auth_hash = &auth_hash_hmac_sha1;
  415                 auth_mode = SHA1;
  416                 break;
  417         case CRYPTO_SHA2_256_HMAC:
  418                 auth_hash = &auth_hash_hmac_sha2_256;
  419                 auth_mode = SHA2_256;
  420                 break;
  421         case CRYPTO_SHA2_384_HMAC:
  422                 auth_hash = &auth_hash_hmac_sha2_384;
  423                 auth_mode = SHA2_384;
  424                 break;
  425         case CRYPTO_SHA2_512_HMAC:
  426                 auth_hash = &auth_hash_hmac_sha2_512;
  427                 auth_mode = SHA2_512;
  428                 break;
  429         default:
  430                 auth_hash = NULL;
  431                 auth_mode = 0;
  432                 break;
  433         }
  434 
  435         switch (csp->csp_cipher_alg) {
  436         case CRYPTO_AES_CBC:
  437                 cipher_mode = CCP_AES_MODE_CBC;
  438                 break;
  439         case CRYPTO_AES_ICM:
  440                 cipher_mode = CCP_AES_MODE_CTR;
  441                 break;
  442         case CRYPTO_AES_NIST_GCM_16:
  443                 cipher_mode = CCP_AES_MODE_GCTR;
  444                 break;
  445         case CRYPTO_AES_XTS:
  446                 cipher_mode = CCP_AES_MODE_XTS;
  447                 break;
  448         default:
  449                 cipher_mode = CCP_AES_MODE_ECB;
  450                 break;
  451         }
  452 
  453         sc = device_get_softc(dev);
  454         mtx_lock(&sc->lock);
  455         if (sc->detaching) {
  456                 mtx_unlock(&sc->lock);
  457                 return (ENXIO);
  458         }
  459 
  460         s = crypto_get_driver_session(cses);
  461 
  462         /* Just grab the first usable queue for now. */
  463         for (q = 0; q < nitems(sc->queues); q++)
  464                 if ((sc->valid_queues & (1 << q)) != 0)
  465                         break;
  466         if (q == nitems(sc->queues)) {
  467                 mtx_unlock(&sc->lock);
  468                 return (ENXIO);
  469         }
  470         s->queue = q;
  471 
  472         switch (csp->csp_mode) {
  473         case CSP_MODE_AEAD:
  474                 s->mode = GCM;
  475                 break;
  476         case CSP_MODE_ETA:
  477                 s->mode = AUTHENC;
  478                 break;
  479         case CSP_MODE_DIGEST:
  480                 s->mode = HMAC;
  481                 break;
  482         case CSP_MODE_CIPHER:
  483                 s->mode = BLKCIPHER;
  484                 break;
  485         }
  486 
  487         if (s->mode == GCM) {
  488                 if (csp->csp_auth_mlen == 0)
  489                         s->gmac.hash_len = AES_GMAC_HASH_LEN;
  490                 else
  491                         s->gmac.hash_len = csp->csp_auth_mlen;
  492         } else if (auth_hash != NULL) {
  493                 s->hmac.auth_hash = auth_hash;
  494                 s->hmac.auth_mode = auth_mode;
  495                 if (csp->csp_auth_mlen == 0)
  496                         s->hmac.hash_len = auth_hash->hashsize;
  497                 else
  498                         s->hmac.hash_len = csp->csp_auth_mlen;
  499                 ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
  500         }
  501         if (cipher_mode != CCP_AES_MODE_ECB) {
  502                 s->blkcipher.cipher_mode = cipher_mode;
  503                 if (csp->csp_cipher_key != NULL)
  504                         ccp_aes_setkey(s, csp->csp_cipher_alg,
  505                             csp->csp_cipher_key, csp->csp_cipher_klen);
  506         }
  507 
  508         s->active = true;
  509         mtx_unlock(&sc->lock);
  510 
  511         return (0);
  512 }
  513 
  514 static void
  515 ccp_freesession(device_t dev, crypto_session_t cses)
  516 {
  517         struct ccp_session *s;
  518 
  519         s = crypto_get_driver_session(cses);
  520 
  521         if (s->pending != 0)
  522                 device_printf(dev,
  523                     "session %p freed with %d pending requests\n", s,
  524                     s->pending);
  525         s->active = false;
  526 }
  527 
  528 static int
  529 ccp_process(device_t dev, struct cryptop *crp, int hint)
  530 {
  531         const struct crypto_session_params *csp;
  532         struct ccp_softc *sc;
  533         struct ccp_queue *qp;
  534         struct ccp_session *s;
  535         int error;
  536         bool qpheld;
  537 
  538         qpheld = false;
  539         qp = NULL;
  540 
  541         csp = crypto_get_params(crp->crp_session);
  542         s = crypto_get_driver_session(crp->crp_session);
  543         sc = device_get_softc(dev);
  544         mtx_lock(&sc->lock);
  545         qp = &sc->queues[s->queue];
  546         mtx_unlock(&sc->lock);
  547         error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
  548         if (error != 0)
  549                 goto out;
  550         qpheld = true;
  551 
  552         error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf);
  553         if (error != 0)
  554                 goto out;
  555 
  556         if (crp->crp_auth_key != NULL) {
  557                 KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
  558                 ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
  559         }
  560         if (crp->crp_cipher_key != NULL)
  561                 ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
  562                     csp->csp_cipher_klen);
  563 
  564         switch (s->mode) {
  565         case HMAC:
  566                 if (s->pending != 0) {
  567                         error = EAGAIN;
  568                         break;
  569                 }
  570                 error = ccp_hmac(qp, s, crp);
  571                 break;
  572         case BLKCIPHER:
  573                 if (s->pending != 0) {
  574                         error = EAGAIN;
  575                         break;
  576                 }
  577                 error = ccp_blkcipher(qp, s, crp);
  578                 break;
  579         case AUTHENC:
  580                 if (s->pending != 0) {
  581                         error = EAGAIN;
  582                         break;
  583                 }
  584                 error = ccp_authenc(qp, s, crp);
  585                 break;
  586         case GCM:
  587                 if (s->pending != 0) {
  588                         error = EAGAIN;
  589                         break;
  590                 }
  591                 error = ccp_gcm(qp, s, crp);
  592                 break;
  593         }
  594 
  595         if (error == 0)
  596                 s->pending++;
  597 
  598 out:
  599         if (qpheld) {
  600                 if (error != 0) {
  601                         /*
  602                          * Squash EAGAIN so callers don't uselessly and
  603                          * expensively retry if the ring was full.
  604                          */
  605                         if (error == EAGAIN)
  606                                 error = ENOMEM;
  607                         ccp_queue_abort(qp);
  608                 } else
  609                         ccp_queue_release(qp);
  610         }
  611 
  612         if (error != 0) {
  613                 DPRINTF(dev, "%s: early error:%d\n", __func__, error);
  614                 crp->crp_etype = error;
  615                 crypto_done(crp);
  616         }
  617         return (0);
  618 }
  619 
  620 static device_method_t ccp_methods[] = {
  621         DEVMETHOD(device_probe,         ccp_probe),
  622         DEVMETHOD(device_attach,        ccp_attach),
  623         DEVMETHOD(device_detach,        ccp_detach),
  624 
  625         DEVMETHOD(cryptodev_probesession, ccp_probesession),
  626         DEVMETHOD(cryptodev_newsession, ccp_newsession),
  627         DEVMETHOD(cryptodev_freesession, ccp_freesession),
  628         DEVMETHOD(cryptodev_process,    ccp_process),
  629 
  630         DEVMETHOD_END
  631 };
  632 
  633 static driver_t ccp_driver = {
  634         "ccp",
  635         ccp_methods,
  636         sizeof(struct ccp_softc)
  637 };
  638 
  639 DRIVER_MODULE(ccp, pci, ccp_driver, NULL, NULL);
  640 MODULE_VERSION(ccp, 1);
  641 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
  642 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
  643 #if 0   /* There are enough known issues that we shouldn't load automatically */
  644 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
  645     nitems(ccp_ids));
  646 #endif
  647 
  648 static int
  649 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
  650 {
  651         struct ccp_softc *sc;
  652 
  653         mtx_assert(&qp->cq_lock, MA_OWNED);
  654         sc = qp->cq_softc;
  655 
  656         if (n < 1 || n >= (1 << sc->ring_size_order))
  657                 return (EINVAL);
  658 
  659         while (true) {
  660                 if (ccp_queue_get_ring_space(qp) >= n)
  661                         return (0);
  662                 if ((mflags & M_WAITOK) == 0)
  663                         return (EAGAIN);
  664                 qp->cq_waiting = true;
  665                 msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
  666         }
  667 }
  668 
  669 int
  670 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
  671 {
  672         int error;
  673 
  674         mtx_lock(&qp->cq_lock);
  675         qp->cq_acq_tail = qp->cq_tail;
  676         error = ccp_queue_reserve_space(qp, n, mflags);
  677         if (error != 0)
  678                 mtx_unlock(&qp->cq_lock);
  679         return (error);
  680 }
  681 
  682 void
  683 ccp_queue_release(struct ccp_queue *qp)
  684 {
  685 
  686         mtx_assert(&qp->cq_lock, MA_OWNED);
  687         if (qp->cq_tail != qp->cq_acq_tail) {
  688                 wmb();
  689                 ccp_queue_write_tail(qp);
  690         }
  691         mtx_unlock(&qp->cq_lock);
  692 }
  693 
  694 void
  695 ccp_queue_abort(struct ccp_queue *qp)
  696 {
  697         unsigned i;
  698 
  699         mtx_assert(&qp->cq_lock, MA_OWNED);
  700 
  701         /* Wipe out any descriptors associated with this aborted txn. */
  702         for (i = qp->cq_acq_tail; i != qp->cq_tail;
  703             i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
  704                 memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
  705         }
  706         qp->cq_tail = qp->cq_acq_tail;
  707 
  708         mtx_unlock(&qp->cq_lock);
  709 }
  710 
  711 #ifdef DDB
  712 #define _db_show_lock(lo)       LOCK_CLASS(lo)->lc_ddb_show(lo)
  713 #define db_show_lock(lk)        _db_show_lock(&(lk)->lock_object)
  714 static void
  715 db_show_ccp_sc(struct ccp_softc *sc)
  716 {
  717 
  718         db_printf("ccp softc at %p\n", sc);
  719         db_printf(" cid: %d\n", (int)sc->cid);
  720 
  721         db_printf(" lock: ");
  722         db_show_lock(&sc->lock);
  723 
  724         db_printf(" detaching: %d\n", (int)sc->detaching);
  725         db_printf(" ring_size_order: %u\n", sc->ring_size_order);
  726 
  727         db_printf(" hw_version: %d\n", (int)sc->hw_version);
  728         db_printf(" hw_features: %b\n", (int)sc->hw_features,
  729             "\2\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
  730             "\11SHA\0103DES\07AES");
  731 
  732         db_printf(" hw status:\n");
  733         db_ccp_show_hw(sc);
  734 }
  735 
  736 static void
  737 db_show_ccp_qp(struct ccp_queue *qp)
  738 {
  739 
  740         db_printf(" lock: ");
  741         db_show_lock(&qp->cq_lock);
  742 
  743         db_printf(" cq_qindex: %u\n", qp->cq_qindex);
  744         db_printf(" cq_softc: %p\n", qp->cq_softc);
  745 
  746         db_printf(" head: %u\n", qp->cq_head);
  747         db_printf(" tail: %u\n", qp->cq_tail);
  748         db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
  749         db_printf(" desc_ring: %p\n", qp->desc_ring);
  750         db_printf(" completions_ring: %p\n", qp->completions_ring);
  751         db_printf(" descriptors (phys): 0x%jx\n",
  752             (uintmax_t)qp->desc_ring_bus_addr);
  753 
  754         db_printf(" hw status:\n");
  755         db_ccp_show_queue_hw(qp);
  756 }
  757 
  758 DB_SHOW_COMMAND(ccp, db_show_ccp)
  759 {
  760         struct ccp_softc *sc;
  761         unsigned unit, qindex;
  762 
  763         if (!have_addr)
  764                 goto usage;
  765 
  766         unit = (unsigned)addr;
  767 
  768         sc = devclass_get_softc(devclass_find("ccp"), unit);
  769         if (sc == NULL) {
  770                 db_printf("No such device ccp%u\n", unit);
  771                 goto usage;
  772         }
  773 
  774         if (count == -1) {
  775                 db_show_ccp_sc(sc);
  776                 return;
  777         }
  778 
  779         qindex = (unsigned)count;
  780         if (qindex >= nitems(sc->queues)) {
  781                 db_printf("No such queue %u\n", qindex);
  782                 goto usage;
  783         }
  784         db_show_ccp_qp(&sc->queues[qindex]);
  785         return;
  786 
  787 usage:
  788         db_printf("usage: show ccp <unit>[,<qindex>]\n");
  789         return;
  790 }
  791 #endif /* DDB */
Cache object: 7a15ac9235da8fe41b33f58267787bd5
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/crypto/ccp/ccp.c

FreeBSD/Linux Kernel Cross Reference
sys/crypto/ccp/ccp.c
