The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/ubsec/ubsec.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
    2 
    3 /*-
    4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
    5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
    6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
    7  *
    8  * All rights reserved.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 3. All advertising materials mentioning features or use of this software
   19  *    must display the following acknowledgement:
   20  *      This product includes software developed by Jason L. Wright
   21  * 4. The name of the author may not be used to endorse or promote products
   22  *    derived from this software without specific prior written permission.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   25  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   27  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
   28  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   29  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   30  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
   32  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
   33  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   34  * POSSIBILITY OF SUCH DAMAGE.
   35  *
   36  * Effort sponsored in part by the Defense Advanced Research Projects
   37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
   38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
   39  */
   40 
   41 #include <sys/cdefs.h>
   42 __FBSDID("$FreeBSD$");
   43 
   44 /*
   45  * uBsec 5[56]01, 58xx hardware crypto accelerator
   46  */
   47 
   48 #include "opt_ubsec.h"
   49 
   50 #include <sys/param.h>
   51 #include <sys/systm.h>
   52 #include <sys/proc.h>
   53 #include <sys/errno.h>
   54 #include <sys/malloc.h>
   55 #include <sys/kernel.h>
   56 #include <sys/module.h>
   57 #include <sys/mbuf.h>
   58 #include <sys/lock.h>
   59 #include <sys/mutex.h>
   60 #include <sys/sysctl.h>
   61 #include <sys/endian.h>
   62 
   63 #include <vm/vm.h>
   64 #include <vm/pmap.h>
   65 
   66 #include <machine/bus.h>
   67 #include <machine/resource.h>
   68 #include <sys/bus.h>
   69 #include <sys/rman.h>
   70 
   71 #include <crypto/sha1.h>
   72 #include <opencrypto/cryptodev.h>
   73 #include <opencrypto/cryptosoft.h>
   74 #include <sys/md5.h>
   75 #include <sys/random.h>
   76 #include <sys/kobj.h>
   77 
   78 #include "cryptodev_if.h"
   79 
   80 #include <dev/pci/pcivar.h>
   81 #include <dev/pci/pcireg.h>
   82 
   83 /* grr, #defines for gratuitous incompatibility in queue.h */
   84 #define SIMPLEQ_HEAD            STAILQ_HEAD
   85 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
   86 #define SIMPLEQ_INIT            STAILQ_INIT
   87 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
   88 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
   89 #define SIMPLEQ_FIRST           STAILQ_FIRST
   90 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD
   91 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
   92 /* ditto for endian.h */
   93 #define letoh16(x)              le16toh(x)
   94 #define letoh32(x)              le32toh(x)
   95 
   96 #ifdef UBSEC_RNDTEST
   97 #include <dev/rndtest/rndtest.h>
   98 #endif
   99 #include <dev/ubsec/ubsecreg.h>
  100 #include <dev/ubsec/ubsecvar.h>
  101 
  102 /*
  103  * Prototypes and count for the pci_device structure
  104  */
  105 static  int ubsec_probe(device_t);
  106 static  int ubsec_attach(device_t);
  107 static  int ubsec_detach(device_t);
  108 static  int ubsec_suspend(device_t);
  109 static  int ubsec_resume(device_t);
  110 static  void ubsec_shutdown(device_t);
  111 
  112 static  int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
  113 static  int ubsec_freesession(device_t, u_int64_t);
  114 static  int ubsec_process(device_t, struct cryptop *, int);
  115 static  int ubsec_kprocess(device_t, struct cryptkop *, int);
  116 
  117 static device_method_t ubsec_methods[] = {
  118         /* Device interface */
  119         DEVMETHOD(device_probe,         ubsec_probe),
  120         DEVMETHOD(device_attach,        ubsec_attach),
  121         DEVMETHOD(device_detach,        ubsec_detach),
  122         DEVMETHOD(device_suspend,       ubsec_suspend),
  123         DEVMETHOD(device_resume,        ubsec_resume),
  124         DEVMETHOD(device_shutdown,      ubsec_shutdown),
  125 
  126         /* bus interface */
  127         DEVMETHOD(bus_print_child,      bus_generic_print_child),
  128         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
  129 
  130         /* crypto device methods */
  131         DEVMETHOD(cryptodev_newsession, ubsec_newsession),
  132         DEVMETHOD(cryptodev_freesession,ubsec_freesession),
  133         DEVMETHOD(cryptodev_process,    ubsec_process),
  134         DEVMETHOD(cryptodev_kprocess,   ubsec_kprocess),
  135 
  136         { 0, 0 }
  137 };
  138 static driver_t ubsec_driver = {
  139         "ubsec",
  140         ubsec_methods,
  141         sizeof (struct ubsec_softc)
  142 };
  143 static devclass_t ubsec_devclass;
  144 
  145 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
  146 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
  147 #ifdef UBSEC_RNDTEST
  148 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
  149 #endif
  150 
  151 static  void ubsec_intr(void *);
  152 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
  153 static  void ubsec_feed(struct ubsec_softc *);
  154 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
  155 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
  156 static  int ubsec_feed2(struct ubsec_softc *);
  157 static  void ubsec_rng(void *);
  158 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
  159                              struct ubsec_dma_alloc *, int);
  160 #define ubsec_dma_sync(_dma, _flags) \
  161         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
  162 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
  163 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
  164 
  165 static  void ubsec_reset_board(struct ubsec_softc *sc);
  166 static  void ubsec_init_board(struct ubsec_softc *sc);
  167 static  void ubsec_init_pciregs(device_t dev);
  168 static  void ubsec_totalreset(struct ubsec_softc *sc);
  169 
  170 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
  171 
  172 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
  173 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
  174 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
  175 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
  176 static  int ubsec_ksigbits(struct crparam *);
  177 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
  178 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
  179 
  180 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
  181 
  182 #ifdef UBSEC_DEBUG
  183 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
  184 static  void ubsec_dump_mcr(struct ubsec_mcr *);
  185 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
  186 
  187 static  int ubsec_debug = 0;
  188 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
  189             0, "control debugging msgs");
  190 #endif
  191 
  192 #define READ_REG(sc,r) \
  193         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
  194 
  195 #define WRITE_REG(sc,reg,val) \
  196         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
  197 
  198 #define SWAP32(x) (x) = htole32(ntohl((x)))
  199 #define HTOLE32(x) (x) = htole32(x)
  200 
  201 struct ubsec_stats ubsecstats;
  202 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
  203             ubsec_stats, "driver statistics");
  204 
  205 static int
  206 ubsec_probe(device_t dev)
  207 {
  208         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
  209             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
  210              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
  211                 return (BUS_PROBE_DEFAULT);
  212         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
  213             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
  214              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
  215                 return (BUS_PROBE_DEFAULT);
  216         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
  217             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
  218              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
  219              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
  220              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
  221              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
  222              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
  223              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823
  224              ))
  225                 return (BUS_PROBE_DEFAULT);
  226         return (ENXIO);
  227 }
  228 
  229 static const char*
  230 ubsec_partname(struct ubsec_softc *sc)
  231 {
  232         /* XXX sprintf numbers when not decoded */
  233         switch (pci_get_vendor(sc->sc_dev)) {
  234         case PCI_VENDOR_BROADCOM:
  235                 switch (pci_get_device(sc->sc_dev)) {
  236                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
  237                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
  238                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
  239                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
  240                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
  241                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
  242                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
  243                 }
  244                 return "Broadcom unknown-part";
  245         case PCI_VENDOR_BLUESTEEL:
  246                 switch (pci_get_device(sc->sc_dev)) {
  247                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
  248                 }
  249                 return "Bluesteel unknown-part";
  250         case PCI_VENDOR_SUN:
  251                 switch (pci_get_device(sc->sc_dev)) {
  252                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
  253                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
  254                 }
  255                 return "Sun unknown-part";
  256         }
  257         return "Unknown-vendor unknown-part";
  258 }
  259 
  260 static void
  261 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
  262 {
  263         random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
  264 }
  265 
  266 static int
  267 ubsec_attach(device_t dev)
  268 {
  269         struct ubsec_softc *sc = device_get_softc(dev);
  270         struct ubsec_dma *dmap;
  271         u_int32_t cmd, i;
  272         int rid;
  273 
  274         bzero(sc, sizeof (*sc));
  275         sc->sc_dev = dev;
  276 
  277         SIMPLEQ_INIT(&sc->sc_queue);
  278         SIMPLEQ_INIT(&sc->sc_qchip);
  279         SIMPLEQ_INIT(&sc->sc_queue2);
  280         SIMPLEQ_INIT(&sc->sc_qchip2);
  281         SIMPLEQ_INIT(&sc->sc_q2free);
  282 
  283         /* XXX handle power management */
  284 
  285         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
  286 
  287         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
  288             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
  289                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
  290 
  291         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
  292             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
  293              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
  294                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
  295 
  296         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
  297             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
  298                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
  299                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
  300 
  301         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
  302              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
  303               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
  304               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) ||
  305             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
  306              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
  307               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
  308                 /* NB: the 5821/5822 defines some additional status bits */
  309                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
  310                     BS_STAT_MCR2_ALLEMPTY;
  311                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
  312                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
  313         }
  314 
  315         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
  316         cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
  317         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
  318         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
  319 
  320         if (!(cmd & PCIM_CMD_MEMEN)) {
  321                 device_printf(dev, "failed to enable memory mapping\n");
  322                 goto bad;
  323         }
  324 
  325         if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
  326                 device_printf(dev, "failed to enable bus mastering\n");
  327                 goto bad;
  328         }
  329 
  330         /*
  331          * Setup memory-mapping of PCI registers.
  332          */
  333         rid = BS_BAR;
  334         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
  335                                            RF_ACTIVE);
  336         if (sc->sc_sr == NULL) {
  337                 device_printf(dev, "cannot map register space\n");
  338                 goto bad;
  339         }
  340         sc->sc_st = rman_get_bustag(sc->sc_sr);
  341         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
  342 
  343         /*
  344          * Arrange interrupt line.
  345          */
  346         rid = 0;
  347         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
  348                                             RF_SHAREABLE|RF_ACTIVE);
  349         if (sc->sc_irq == NULL) {
  350                 device_printf(dev, "could not map interrupt\n");
  351                 goto bad1;
  352         }
  353         /*
  354          * NB: Network code assumes we are blocked with splimp()
  355          *     so make sure the IRQ is mapped appropriately.
  356          */
  357         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
  358                            NULL, ubsec_intr, sc, &sc->sc_ih)) {
  359                 device_printf(dev, "could not establish interrupt\n");
  360                 goto bad2;
  361         }
  362 
  363         sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
  364         if (sc->sc_cid < 0) {
  365                 device_printf(dev, "could not get crypto driver id\n");
  366                 goto bad3;
  367         }
  368 
  369         /*
  370          * Setup DMA descriptor area.
  371          */
  372         if (bus_dma_tag_create(NULL,                    /* parent */
  373                                1, 0,                    /* alignment, bounds */
  374                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
  375                                BUS_SPACE_MAXADDR,       /* highaddr */
  376                                NULL, NULL,              /* filter, filterarg */
  377                                0x3ffff,                 /* maxsize */
  378                                UBS_MAX_SCATTER,         /* nsegments */
  379                                0xffff,                  /* maxsegsize */
  380                                BUS_DMA_ALLOCNOW,        /* flags */
  381                                NULL, NULL,              /* lockfunc, lockarg */
  382                                &sc->sc_dmat)) {
  383                 device_printf(dev, "cannot allocate DMA tag\n");
  384                 goto bad4;
  385         }
  386         SIMPLEQ_INIT(&sc->sc_freequeue);
  387         dmap = sc->sc_dmaa;
  388         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
  389                 struct ubsec_q *q;
  390 
  391                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
  392                     M_DEVBUF, M_NOWAIT);
  393                 if (q == NULL) {
  394                         device_printf(dev, "cannot allocate queue buffers\n");
  395                         break;
  396                 }
  397 
  398                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
  399                     &dmap->d_alloc, 0)) {
  400                         device_printf(dev, "cannot allocate dma buffers\n");
  401                         free(q, M_DEVBUF);
  402                         break;
  403                 }
  404                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
  405 
  406                 q->q_dma = dmap;
  407                 sc->sc_queuea[i] = q;
  408 
  409                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
  410         }
  411         mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
  412                 "mcr1 operations", MTX_DEF);
  413         mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
  414                 "mcr1 free q", MTX_DEF);
  415 
  416         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
  417 
  418         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
  419         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
  420         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
  421         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
  422 
  423         /*
  424          * Reset Broadcom chip
  425          */
  426         ubsec_reset_board(sc);
  427 
  428         /*
  429          * Init Broadcom specific PCI settings
  430          */
  431         ubsec_init_pciregs(dev);
  432 
  433         /*
  434          * Init Broadcom chip
  435          */
  436         ubsec_init_board(sc);
  437 
  438 #ifndef UBSEC_NO_RNG
  439         if (sc->sc_flags & UBS_FLAGS_RNG) {
  440                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
  441 #ifdef UBSEC_RNDTEST
  442                 sc->sc_rndtest = rndtest_attach(dev);
  443                 if (sc->sc_rndtest)
  444                         sc->sc_harvest = rndtest_harvest;
  445                 else
  446                         sc->sc_harvest = default_harvest;
  447 #else
  448                 sc->sc_harvest = default_harvest;
  449 #endif
  450 
  451                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
  452                     &sc->sc_rng.rng_q.q_mcr, 0))
  453                         goto skip_rng;
  454 
  455                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
  456                     &sc->sc_rng.rng_q.q_ctx, 0)) {
  457                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
  458                         goto skip_rng;
  459                 }
  460 
  461                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
  462                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
  463                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
  464                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
  465                         goto skip_rng;
  466                 }
  467 
  468                 if (hz >= 100)
  469                         sc->sc_rnghz = hz / 100;
  470                 else
  471                         sc->sc_rnghz = 1;
  472                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
  473                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
  474 skip_rng:
  475         ;
  476         }
  477 #endif /* UBSEC_NO_RNG */
  478         mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
  479                 "mcr2 operations", MTX_DEF);
  480 
  481         if (sc->sc_flags & UBS_FLAGS_KEY) {
  482                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
  483 
  484                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
  485 #if 0
  486                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
  487 #endif
  488         }
  489         return (0);
  490 bad4:
  491         crypto_unregister_all(sc->sc_cid);
  492 bad3:
  493         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
  494 bad2:
  495         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
  496 bad1:
  497         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
  498 bad:
  499         return (ENXIO);
  500 }
  501 
  502 /*
  503  * Detach a device that successfully probed.
  504  */
  505 static int
  506 ubsec_detach(device_t dev)
  507 {
  508         struct ubsec_softc *sc = device_get_softc(dev);
  509 
  510         /* XXX wait/abort active ops */
  511 
  512         /* disable interrupts */
  513         WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
  514                 (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
  515 
  516         callout_stop(&sc->sc_rngto);
  517 
  518         crypto_unregister_all(sc->sc_cid);
  519 
  520 #ifdef UBSEC_RNDTEST
  521         if (sc->sc_rndtest)
  522                 rndtest_detach(sc->sc_rndtest);
  523 #endif
  524 
  525         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
  526                 struct ubsec_q *q;
  527 
  528                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
  529                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
  530                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
  531                 free(q, M_DEVBUF);
  532         }
  533         mtx_destroy(&sc->sc_mcr1lock);
  534         mtx_destroy(&sc->sc_freeqlock);
  535 #ifndef UBSEC_NO_RNG
  536         if (sc->sc_flags & UBS_FLAGS_RNG) {
  537                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
  538                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
  539                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
  540         }
  541 #endif /* UBSEC_NO_RNG */
  542         mtx_destroy(&sc->sc_mcr2lock);
  543 
  544         bus_generic_detach(dev);
  545         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
  546         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
  547 
  548         bus_dma_tag_destroy(sc->sc_dmat);
  549         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
  550 
  551         return (0);
  552 }
  553 
  554 /*
  555  * Stop all chip i/o so that the kernel's probe routines don't
  556  * get confused by errant DMAs when rebooting.
  557  */
  558 static void
  559 ubsec_shutdown(device_t dev)
  560 {
  561 #ifdef notyet
  562         ubsec_stop(device_get_softc(dev));
  563 #endif
  564 }
  565 
  566 /*
  567  * Device suspend routine.
  568  */
  569 static int
  570 ubsec_suspend(device_t dev)
  571 {
  572         struct ubsec_softc *sc = device_get_softc(dev);
  573 
  574 #ifdef notyet
  575         /* XXX stop the device and save PCI settings */
  576 #endif
  577         sc->sc_suspended = 1;
  578 
  579         return (0);
  580 }
  581 
  582 static int
  583 ubsec_resume(device_t dev)
  584 {
  585         struct ubsec_softc *sc = device_get_softc(dev);
  586 
  587 #ifdef notyet
  588         /* XXX retore PCI settings and start the device */
  589 #endif
  590         sc->sc_suspended = 0;
  591         return (0);
  592 }
  593 
  594 /*
  595  * UBSEC Interrupt routine
  596  */
  597 static void
  598 ubsec_intr(void *arg)
  599 {
  600         struct ubsec_softc *sc = arg;
  601         volatile u_int32_t stat;
  602         struct ubsec_q *q;
  603         struct ubsec_dma *dmap;
  604         int npkts = 0, i;
  605 
  606         stat = READ_REG(sc, BS_STAT);
  607         stat &= sc->sc_statmask;
  608         if (stat == 0)
  609                 return;
  610 
  611         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
  612 
  613         /*
  614          * Check to see if we have any packets waiting for us
  615          */
  616         if ((stat & BS_STAT_MCR1_DONE)) {
  617                 mtx_lock(&sc->sc_mcr1lock);
  618                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
  619                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
  620                         dmap = q->q_dma;
  621 
  622                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
  623                                 break;
  624 
  625                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
  626 
  627                         npkts = q->q_nstacked_mcrs;
  628                         sc->sc_nqchip -= 1+npkts;
  629                         /*
  630                          * search for further sc_qchip ubsec_q's that share
  631                          * the same MCR, and complete them too, they must be
  632                          * at the top.
  633                          */
  634                         for (i = 0; i < npkts; i++) {
  635                                 if(q->q_stacked_mcr[i]) {
  636                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
  637                                 } else {
  638                                         break;
  639                                 }
  640                         }
  641                         ubsec_callback(sc, q);
  642                 }
  643                 /*
  644                  * Don't send any more packet to chip if there has been
  645                  * a DMAERR.
  646                  */
  647                 if (!(stat & BS_STAT_DMAERR))
  648                         ubsec_feed(sc);
  649                 mtx_unlock(&sc->sc_mcr1lock);
  650         }
  651 
  652         /*
  653          * Check to see if we have any key setups/rng's waiting for us
  654          */
  655         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
  656             (stat & BS_STAT_MCR2_DONE)) {
  657                 struct ubsec_q2 *q2;
  658                 struct ubsec_mcr *mcr;
  659 
  660                 mtx_lock(&sc->sc_mcr2lock);
  661                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
  662                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
  663 
  664                         ubsec_dma_sync(&q2->q_mcr,
  665                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
  666 
  667                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
  668                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
  669                                 ubsec_dma_sync(&q2->q_mcr,
  670                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
  671                                 break;
  672                         }
  673                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
  674                         ubsec_callback2(sc, q2);
  675                         /*
  676                          * Don't send any more packet to chip if there has been
  677                          * a DMAERR.
  678                          */
  679                         if (!(stat & BS_STAT_DMAERR))
  680                                 ubsec_feed2(sc);
  681                 }
  682                 mtx_unlock(&sc->sc_mcr2lock);
  683         }
  684 
  685         /*
  686          * Check to see if we got any DMA Error
  687          */
  688         if (stat & BS_STAT_DMAERR) {
  689 #ifdef UBSEC_DEBUG
  690                 if (ubsec_debug) {
  691                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
  692 
  693                         printf("dmaerr %s@%08x\n",
  694                             (a & BS_ERR_READ) ? "read" : "write",
  695                             a & BS_ERR_ADDR);
  696                 }
  697 #endif /* UBSEC_DEBUG */
  698                 ubsecstats.hst_dmaerr++;
  699                 mtx_lock(&sc->sc_mcr1lock);
  700                 ubsec_totalreset(sc);
  701                 ubsec_feed(sc);
  702                 mtx_unlock(&sc->sc_mcr1lock);
  703         }
  704 
  705         if (sc->sc_needwakeup) {                /* XXX check high watermark */
  706                 int wakeup;
  707 
  708                 mtx_lock(&sc->sc_freeqlock);
  709                 wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
  710 #ifdef UBSEC_DEBUG
  711                 if (ubsec_debug)
  712                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
  713                                 sc->sc_needwakeup);
  714 #endif /* UBSEC_DEBUG */
  715                 sc->sc_needwakeup &= ~wakeup;
  716                 mtx_unlock(&sc->sc_freeqlock);
  717                 crypto_unblock(sc->sc_cid, wakeup);
  718         }
  719 }
  720 
  721 /*
  722  * ubsec_feed() - aggregate and post requests to chip
  723  */
  724 static void
  725 ubsec_feed(struct ubsec_softc *sc)
  726 {
  727         struct ubsec_q *q, *q2;
  728         int npkts, i;
  729         void *v;
  730         u_int32_t stat;
  731 
  732         /*
  733          * Decide how many ops to combine in a single MCR.  We cannot
  734          * aggregate more than UBS_MAX_AGGR because this is the number
  735          * of slots defined in the data structure.  Note that
  736          * aggregation only happens if ops are marked batch'able.
  737          * Aggregating ops reduces the number of interrupts to the host
  738          * but also (potentially) increases the latency for processing
  739          * completed ops as we only get an interrupt when all aggregated
  740          * ops have completed.
  741          */
  742         if (sc->sc_nqueue == 0)
  743                 return;
  744         if (sc->sc_nqueue > 1) {
  745                 npkts = 0;
  746                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
  747                         npkts++;
  748                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
  749                                 break;
  750                 }
  751         } else
  752                 npkts = 1;
  753         /*
  754          * Check device status before going any further.
  755          */
  756         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
  757                 if (stat & BS_STAT_DMAERR) {
  758                         ubsec_totalreset(sc);
  759                         ubsecstats.hst_dmaerr++;
  760                 } else
  761                         ubsecstats.hst_mcr1full++;
  762                 return;
  763         }
  764         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
  765                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
  766         if (npkts > UBS_MAX_AGGR)
  767                 npkts = UBS_MAX_AGGR;
  768         if (npkts < 2)                          /* special case 1 op */
  769                 goto feed1;
  770 
  771         ubsecstats.hst_totbatch += npkts-1;
  772 #ifdef UBSEC_DEBUG
  773         if (ubsec_debug)
  774                 printf("merging %d records\n", npkts);
  775 #endif /* UBSEC_DEBUG */
  776 
  777         q = SIMPLEQ_FIRST(&sc->sc_queue);
  778         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
  779         --sc->sc_nqueue;
  780 
  781         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
  782         if (q->q_dst_map != NULL)
  783                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
  784 
  785         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
  786 
  787         for (i = 0; i < q->q_nstacked_mcrs; i++) {
  788                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
  789                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
  790                     BUS_DMASYNC_PREWRITE);
  791                 if (q2->q_dst_map != NULL)
  792                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
  793                             BUS_DMASYNC_PREREAD);
  794                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
  795                 --sc->sc_nqueue;
  796 
  797                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
  798                     sizeof(struct ubsec_mcr_add));
  799                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
  800                 q->q_stacked_mcr[i] = q2;
  801         }
  802         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
  803         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
  804         sc->sc_nqchip += npkts;
  805         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
  806                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
  807         ubsec_dma_sync(&q->q_dma->d_alloc,
  808             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  809         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
  810             offsetof(struct ubsec_dmachunk, d_mcr));
  811         return;
  812 feed1:
  813         q = SIMPLEQ_FIRST(&sc->sc_queue);
  814 
  815         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
  816         if (q->q_dst_map != NULL)
  817                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
  818         ubsec_dma_sync(&q->q_dma->d_alloc,
  819             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  820 
  821         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
  822             offsetof(struct ubsec_dmachunk, d_mcr));
  823 #ifdef UBSEC_DEBUG
  824         if (ubsec_debug)
  825                 printf("feed1: q->chip %p %08x stat %08x\n",
  826                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
  827                       stat);
  828 #endif /* UBSEC_DEBUG */
  829         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
  830         --sc->sc_nqueue;
  831         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
  832         sc->sc_nqchip++;
  833         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
  834                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
  835         return;
  836 }
  837 
  838 static void
  839 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
  840 {
  841 
  842         /* Go ahead and compute key in ubsec's byte order */
  843         if (algo == CRYPTO_DES_CBC) {
  844                 bcopy(key, &ses->ses_deskey[0], 8);
  845                 bcopy(key, &ses->ses_deskey[2], 8);
  846                 bcopy(key, &ses->ses_deskey[4], 8);
  847         } else
  848                 bcopy(key, ses->ses_deskey, 24);
  849 
  850         SWAP32(ses->ses_deskey[0]);
  851         SWAP32(ses->ses_deskey[1]);
  852         SWAP32(ses->ses_deskey[2]);
  853         SWAP32(ses->ses_deskey[3]);
  854         SWAP32(ses->ses_deskey[4]);
  855         SWAP32(ses->ses_deskey[5]);
  856 }
  857 
  858 static void
  859 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
  860 {
  861         MD5_CTX md5ctx;
  862         SHA1_CTX sha1ctx;
  863         int i;
  864 
  865         for (i = 0; i < klen; i++)
  866                 key[i] ^= HMAC_IPAD_VAL;
  867 
  868         if (algo == CRYPTO_MD5_HMAC) {
  869                 MD5Init(&md5ctx);
  870                 MD5Update(&md5ctx, key, klen);
  871                 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
  872                 bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
  873         } else {
  874                 SHA1Init(&sha1ctx);
  875                 SHA1Update(&sha1ctx, key, klen);
  876                 SHA1Update(&sha1ctx, hmac_ipad_buffer,
  877                     SHA1_HMAC_BLOCK_LEN - klen);
  878                 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
  879         }
  880 
  881         for (i = 0; i < klen; i++)
  882                 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
  883 
  884         if (algo == CRYPTO_MD5_HMAC) {
  885                 MD5Init(&md5ctx);
  886                 MD5Update(&md5ctx, key, klen);
  887                 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
  888                 bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
  889         } else {
  890                 SHA1Init(&sha1ctx);
  891                 SHA1Update(&sha1ctx, key, klen);
  892                 SHA1Update(&sha1ctx, hmac_opad_buffer,
  893                     SHA1_HMAC_BLOCK_LEN - klen);
  894                 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
  895         }
  896 
  897         for (i = 0; i < klen; i++)
  898                 key[i] ^= HMAC_OPAD_VAL;
  899 }
  900 
  901 /*
  902  * Allocate a new 'session' and return an encoded session id.  'sidp'
  903  * contains our registration id, and should contain an encoded session
  904  * id on successful allocation.
  905  */
  906 static int
  907 ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
  908 {
  909         struct ubsec_softc *sc = device_get_softc(dev);
  910         struct cryptoini *c, *encini = NULL, *macini = NULL;
  911         struct ubsec_session *ses = NULL;
  912         int sesn;
  913 
  914         if (sidp == NULL || cri == NULL || sc == NULL)
  915                 return (EINVAL);
  916 
  917         for (c = cri; c != NULL; c = c->cri_next) {
  918                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
  919                     c->cri_alg == CRYPTO_SHA1_HMAC) {
  920                         if (macini)
  921                                 return (EINVAL);
  922                         macini = c;
  923                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
  924                     c->cri_alg == CRYPTO_3DES_CBC) {
  925                         if (encini)
  926                                 return (EINVAL);
  927                         encini = c;
  928                 } else
  929                         return (EINVAL);
  930         }
  931         if (encini == NULL && macini == NULL)
  932                 return (EINVAL);
  933 
  934         if (sc->sc_sessions == NULL) {
  935                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
  936                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
  937                 if (ses == NULL)
  938                         return (ENOMEM);
  939                 sesn = 0;
  940                 sc->sc_nsessions = 1;
  941         } else {
  942                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
  943                         if (sc->sc_sessions[sesn].ses_used == 0) {
  944                                 ses = &sc->sc_sessions[sesn];
  945                                 break;
  946                         }
  947                 }
  948 
  949                 if (ses == NULL) {
  950                         sesn = sc->sc_nsessions;
  951                         ses = (struct ubsec_session *)malloc((sesn + 1) *
  952                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
  953                         if (ses == NULL)
  954                                 return (ENOMEM);
  955                         bcopy(sc->sc_sessions, ses, sesn *
  956                             sizeof(struct ubsec_session));
  957                         bzero(sc->sc_sessions, sesn *
  958                             sizeof(struct ubsec_session));
  959                         free(sc->sc_sessions, M_DEVBUF);
  960                         sc->sc_sessions = ses;
  961                         ses = &sc->sc_sessions[sesn];
  962                         sc->sc_nsessions++;
  963                 }
  964         }
  965         bzero(ses, sizeof(struct ubsec_session));
  966         ses->ses_used = 1;
  967 
  968         if (encini) {
  969                 /* get an IV, network byte order */
  970                 /* XXX may read fewer than requested */
  971                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
  972 
  973                 if (encini->cri_key != NULL) {
  974                         ubsec_setup_enckey(ses, encini->cri_alg,
  975                             encini->cri_key);
  976                 }
  977         }
  978 
  979         if (macini) {
  980                 ses->ses_mlen = macini->cri_mlen;
  981                 if (ses->ses_mlen == 0) {
  982                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
  983                                 ses->ses_mlen = MD5_HASH_LEN;
  984                         else
  985                                 ses->ses_mlen = SHA1_HASH_LEN;
  986                 }
  987 
  988                 if (macini->cri_key != NULL) {
  989                         ubsec_setup_mackey(ses, macini->cri_alg,
  990                             macini->cri_key, macini->cri_klen / 8);
  991                 }
  992         }
  993 
  994         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
  995         return (0);
  996 }
  997 
  998 /*
  999  * Deallocate a session.
 1000  */
 1001 static int
 1002 ubsec_freesession(device_t dev, u_int64_t tid)
 1003 {
 1004         struct ubsec_softc *sc = device_get_softc(dev);
 1005         int session, ret;
 1006         u_int32_t sid = CRYPTO_SESID2LID(tid);
 1007 
 1008         if (sc == NULL)
 1009                 return (EINVAL);
 1010 
 1011         session = UBSEC_SESSION(sid);
 1012         if (session < sc->sc_nsessions) {
 1013                 bzero(&sc->sc_sessions[session],
 1014                         sizeof(sc->sc_sessions[session]));
 1015                 ret = 0;
 1016         } else
 1017                 ret = EINVAL;
 1018 
 1019         return (ret);
 1020 }
 1021 
 1022 static void
 1023 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 1024 {
 1025         struct ubsec_operand *op = arg;
 1026 
 1027         KASSERT(nsegs <= UBS_MAX_SCATTER,
 1028                 ("Too many DMA segments returned when mapping operand"));
 1029 #ifdef UBSEC_DEBUG
 1030         if (ubsec_debug)
 1031                 printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
 1032                         (u_int) mapsize, nsegs, error);
 1033 #endif
 1034         if (error != 0)
 1035                 return;
 1036         op->mapsize = mapsize;
 1037         op->nsegs = nsegs;
 1038         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 1039 }
 1040 
 1041 static int
 1042 ubsec_process(device_t dev, struct cryptop *crp, int hint)
 1043 {
 1044         struct ubsec_softc *sc = device_get_softc(dev);
 1045         struct ubsec_q *q = NULL;
 1046         int err = 0, i, j, nicealign;
 1047         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 1048         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
 1049         int sskip, dskip, stheend, dtheend;
 1050         int16_t coffset;
 1051         struct ubsec_session *ses;
 1052         struct ubsec_pktctx ctx;
 1053         struct ubsec_dma *dmap = NULL;
 1054 
 1055         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 1056                 ubsecstats.hst_invalid++;
 1057                 return (EINVAL);
 1058         }
 1059         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 1060                 ubsecstats.hst_badsession++;
 1061                 return (EINVAL);
 1062         }
 1063 
 1064         mtx_lock(&sc->sc_freeqlock);
 1065         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 1066                 ubsecstats.hst_queuefull++;
 1067                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 1068                 mtx_unlock(&sc->sc_freeqlock);
 1069                 return (ERESTART);
 1070         }
 1071         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 1072         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 1073         mtx_unlock(&sc->sc_freeqlock);
 1074 
 1075         dmap = q->q_dma; /* Save dma pointer */
 1076         bzero(q, sizeof(struct ubsec_q));
 1077         bzero(&ctx, sizeof(ctx));
 1078 
 1079         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
 1080         q->q_dma = dmap;
 1081         ses = &sc->sc_sessions[q->q_sesn];
 1082 
 1083         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 1084                 q->q_src_m = (struct mbuf *)crp->crp_buf;
 1085                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
 1086         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 1087                 q->q_src_io = (struct uio *)crp->crp_buf;
 1088                 q->q_dst_io = (struct uio *)crp->crp_buf;
 1089         } else {
 1090                 ubsecstats.hst_badflags++;
 1091                 err = EINVAL;
 1092                 goto errout;    /* XXX we don't handle contiguous blocks! */
 1093         }
 1094 
 1095         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
 1096 
 1097         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
 1098         dmap->d_dma->d_mcr.mcr_flags = 0;
 1099         q->q_crp = crp;
 1100 
 1101         crd1 = crp->crp_desc;
 1102         if (crd1 == NULL) {
 1103                 ubsecstats.hst_nodesc++;
 1104                 err = EINVAL;
 1105                 goto errout;
 1106         }
 1107         crd2 = crd1->crd_next;
 1108 
 1109         if (crd2 == NULL) {
 1110                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 1111                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
 1112                         maccrd = crd1;
 1113                         enccrd = NULL;
 1114                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 1115                     crd1->crd_alg == CRYPTO_3DES_CBC) {
 1116                         maccrd = NULL;
 1117                         enccrd = crd1;
 1118                 } else {
 1119                         ubsecstats.hst_badalg++;
 1120                         err = EINVAL;
 1121                         goto errout;
 1122                 }
 1123         } else {
 1124                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 1125                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
 1126                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 1127                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
 1128                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 1129                         maccrd = crd1;
 1130                         enccrd = crd2;
 1131                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 1132                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
 1133                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 1134                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
 1135                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 1136                         enccrd = crd1;
 1137                         maccrd = crd2;
 1138                 } else {
 1139                         /*
 1140                          * We cannot order the ubsec as requested
 1141                          */
 1142                         ubsecstats.hst_badalg++;
 1143                         err = EINVAL;
 1144                         goto errout;
 1145                 }
 1146         }
 1147 
 1148         if (enccrd) {
 1149                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 1150                         ubsec_setup_enckey(ses, enccrd->crd_alg,
 1151                             enccrd->crd_key);
 1152                 }
 1153 
 1154                 encoffset = enccrd->crd_skip;
 1155                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
 1156 
 1157                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 1158                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
 1159 
 1160                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 1161                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 1162                         else {
 1163                                 ctx.pc_iv[0] = ses->ses_iv[0];
 1164                                 ctx.pc_iv[1] = ses->ses_iv[1];
 1165                         }
 1166 
 1167                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 1168                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 1169                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
 1170                         }
 1171                 } else {
 1172                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
 1173 
 1174                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 1175                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 1176                         else {
 1177                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 1178                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
 1179                         }
 1180                 }
 1181 
 1182                 ctx.pc_deskey[0] = ses->ses_deskey[0];
 1183                 ctx.pc_deskey[1] = ses->ses_deskey[1];
 1184                 ctx.pc_deskey[2] = ses->ses_deskey[2];
 1185                 ctx.pc_deskey[3] = ses->ses_deskey[3];
 1186                 ctx.pc_deskey[4] = ses->ses_deskey[4];
 1187                 ctx.pc_deskey[5] = ses->ses_deskey[5];
 1188                 SWAP32(ctx.pc_iv[0]);
 1189                 SWAP32(ctx.pc_iv[1]);
 1190         }
 1191 
 1192         if (maccrd) {
 1193                 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 1194                         ubsec_setup_mackey(ses, maccrd->crd_alg,
 1195                             maccrd->crd_key, maccrd->crd_klen / 8);
 1196                 }
 1197 
 1198                 macoffset = maccrd->crd_skip;
 1199 
 1200                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
 1201                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
 1202                 else
 1203                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
 1204 
 1205                 for (i = 0; i < 5; i++) {
 1206                         ctx.pc_hminner[i] = ses->ses_hminner[i];
 1207                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
 1208 
 1209                         HTOLE32(ctx.pc_hminner[i]);
 1210                         HTOLE32(ctx.pc_hmouter[i]);
 1211                 }
 1212         }
 1213 
 1214         if (enccrd && maccrd) {
 1215                 /*
 1216                  * ubsec cannot handle packets where the end of encryption
 1217                  * and authentication are not the same, or where the
 1218                  * encrypted part begins before the authenticated part.
 1219                  */
 1220                 if ((encoffset + enccrd->crd_len) !=
 1221                     (macoffset + maccrd->crd_len)) {
 1222                         ubsecstats.hst_lenmismatch++;
 1223                         err = EINVAL;
 1224                         goto errout;
 1225                 }
 1226                 if (enccrd->crd_skip < maccrd->crd_skip) {
 1227                         ubsecstats.hst_skipmismatch++;
 1228                         err = EINVAL;
 1229                         goto errout;
 1230                 }
 1231                 sskip = maccrd->crd_skip;
 1232                 cpskip = dskip = enccrd->crd_skip;
 1233                 stheend = maccrd->crd_len;
 1234                 dtheend = enccrd->crd_len;
 1235                 coffset = enccrd->crd_skip - maccrd->crd_skip;
 1236                 cpoffset = cpskip + dtheend;
 1237 #ifdef UBSEC_DEBUG
 1238                 if (ubsec_debug) {
 1239                         printf("mac: skip %d, len %d, inject %d\n",
 1240                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
 1241                         printf("enc: skip %d, len %d, inject %d\n",
 1242                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
 1243                         printf("src: skip %d, len %d\n", sskip, stheend);
 1244                         printf("dst: skip %d, len %d\n", dskip, dtheend);
 1245                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
 1246                             coffset, stheend, cpskip, cpoffset);
 1247                 }
 1248 #endif
 1249         } else {
 1250                 cpskip = dskip = sskip = macoffset + encoffset;
 1251                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
 1252                 cpoffset = cpskip + dtheend;
 1253                 coffset = 0;
 1254         }
 1255         ctx.pc_offset = htole16(coffset >> 2);
 1256 
 1257         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
 1258                 ubsecstats.hst_nomap++;
 1259                 err = ENOMEM;
 1260                 goto errout;
 1261         }
 1262         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 1263                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
 1264                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
 1265                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 1266                         q->q_src_map = NULL;
 1267                         ubsecstats.hst_noload++;
 1268                         err = ENOMEM;
 1269                         goto errout;
 1270                 }
 1271         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 1272                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
 1273                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
 1274                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 1275                         q->q_src_map = NULL;
 1276                         ubsecstats.hst_noload++;
 1277                         err = ENOMEM;
 1278                         goto errout;
 1279                 }
 1280         }
 1281         nicealign = ubsec_dmamap_aligned(&q->q_src);
 1282 
 1283         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
 1284 
 1285 #ifdef UBSEC_DEBUG
 1286         if (ubsec_debug)
 1287                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
 1288 #endif
 1289         for (i = j = 0; i < q->q_src_nsegs; i++) {
 1290                 struct ubsec_pktbuf *pb;
 1291                 bus_size_t packl = q->q_src_segs[i].ds_len;
 1292                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
 1293 
 1294                 if (sskip >= packl) {
 1295                         sskip -= packl;
 1296                         continue;
 1297                 }
 1298 
 1299                 packl -= sskip;
 1300                 packp += sskip;
 1301                 sskip = 0;
 1302 
 1303                 if (packl > 0xfffc) {
 1304                         err = EIO;
 1305                         goto errout;
 1306                 }
 1307 
 1308                 if (j == 0)
 1309                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
 1310                 else
 1311                         pb = &dmap->d_dma->d_sbuf[j - 1];
 1312 
 1313                 pb->pb_addr = htole32(packp);
 1314 
 1315                 if (stheend) {
 1316                         if (packl > stheend) {
 1317                                 pb->pb_len = htole32(stheend);
 1318                                 stheend = 0;
 1319                         } else {
 1320                                 pb->pb_len = htole32(packl);
 1321                                 stheend -= packl;
 1322                         }
 1323                 } else
 1324                         pb->pb_len = htole32(packl);
 1325 
 1326                 if ((i + 1) == q->q_src_nsegs)
 1327                         pb->pb_next = 0;
 1328                 else
 1329                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 1330                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
 1331                 j++;
 1332         }
 1333 
 1334         if (enccrd == NULL && maccrd != NULL) {
 1335                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
 1336                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
 1337                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
 1338                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 1339 #ifdef UBSEC_DEBUG
 1340                 if (ubsec_debug)
 1341                         printf("opkt: %x %x %x\n",
 1342                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
 1343                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
 1344                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
 1345 #endif
 1346         } else {
 1347                 if (crp->crp_flags & CRYPTO_F_IOV) {
 1348                         if (!nicealign) {
 1349                                 ubsecstats.hst_iovmisaligned++;
 1350                                 err = EINVAL;
 1351                                 goto errout;
 1352                         }
 1353                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
 1354                              &q->q_dst_map)) {
 1355                                 ubsecstats.hst_nomap++;
 1356                                 err = ENOMEM;
 1357                                 goto errout;
 1358                         }
 1359                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
 1360                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
 1361                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 1362                                 q->q_dst_map = NULL;
 1363                                 ubsecstats.hst_noload++;
 1364                                 err = ENOMEM;
 1365                                 goto errout;
 1366                         }
 1367                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
 1368                         if (nicealign) {
 1369                                 q->q_dst = q->q_src;
 1370                         } else {
 1371                                 int totlen, len;
 1372                                 struct mbuf *m, *top, **mp;
 1373 
 1374                                 ubsecstats.hst_unaligned++;
 1375                                 totlen = q->q_src_mapsize;
 1376                                 if (totlen >= MINCLSIZE) {
 1377                                         m = m_getcl(M_DONTWAIT, MT_DATA,
 1378                                             q->q_src_m->m_flags & M_PKTHDR);
 1379                                         len = MCLBYTES;
 1380                                 } else if (q->q_src_m->m_flags & M_PKTHDR) {
 1381                                         m = m_gethdr(M_DONTWAIT, MT_DATA);
 1382                                         len = MHLEN;
 1383                                 } else {
 1384                                         m = m_get(M_DONTWAIT, MT_DATA);
 1385                                         len = MLEN;
 1386                                 }
 1387                                 if (m && q->q_src_m->m_flags & M_PKTHDR &&
 1388                                     !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) {
 1389                                         m_free(m);
 1390                                         m = NULL;
 1391                                 }
 1392                                 if (m == NULL) {
 1393                                         ubsecstats.hst_nombuf++;
 1394                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
 1395                                         goto errout;
 1396                                 }
 1397                                 m->m_len = len = min(totlen, len);
 1398                                 totlen -= len;
 1399                                 top = m;
 1400                                 mp = &top;
 1401 
 1402                                 while (totlen > 0) {
 1403                                         if (totlen >= MINCLSIZE) {
 1404                                                 m = m_getcl(M_DONTWAIT,
 1405                                                     MT_DATA, 0);
 1406                                                 len = MCLBYTES;
 1407                                         } else {
 1408                                                 m = m_get(M_DONTWAIT, MT_DATA);
 1409                                                 len = MLEN;
 1410                                         }
 1411                                         if (m == NULL) {
 1412                                                 m_freem(top);
 1413                                                 ubsecstats.hst_nombuf++;
 1414                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
 1415                                                 goto errout;
 1416                                         }
 1417                                         m->m_len = len = min(totlen, len);
 1418                                         totlen -= len;
 1419                                         *mp = m;
 1420                                         mp = &m->m_next;
 1421                                 }
 1422                                 q->q_dst_m = top;
 1423                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
 1424                                     cpskip, cpoffset);
 1425                                 if (bus_dmamap_create(sc->sc_dmat,
 1426                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
 1427                                         ubsecstats.hst_nomap++;
 1428                                         err = ENOMEM;
 1429                                         goto errout;
 1430                                 }
 1431                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
 1432                                     q->q_dst_map, q->q_dst_m,
 1433                                     ubsec_op_cb, &q->q_dst,
 1434                                     BUS_DMA_NOWAIT) != 0) {
 1435                                         bus_dmamap_destroy(sc->sc_dmat,
 1436                                         q->q_dst_map);
 1437                                         q->q_dst_map = NULL;
 1438                                         ubsecstats.hst_noload++;
 1439                                         err = ENOMEM;
 1440                                         goto errout;
 1441                                 }
 1442                         }
 1443                 } else {
 1444                         ubsecstats.hst_badflags++;
 1445                         err = EINVAL;
 1446                         goto errout;
 1447                 }
 1448 
 1449 #ifdef UBSEC_DEBUG
 1450                 if (ubsec_debug)
 1451                         printf("dst skip: %d\n", dskip);
 1452 #endif
 1453                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
 1454                         struct ubsec_pktbuf *pb;
 1455                         bus_size_t packl = q->q_dst_segs[i].ds_len;
 1456                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
 1457 
 1458                         if (dskip >= packl) {
 1459                                 dskip -= packl;
 1460                                 continue;
 1461                         }
 1462 
 1463                         packl -= dskip;
 1464                         packp += dskip;
 1465                         dskip = 0;
 1466 
 1467                         if (packl > 0xfffc) {
 1468                                 err = EIO;
 1469                                 goto errout;
 1470                         }
 1471 
 1472                         if (j == 0)
 1473                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
 1474                         else
 1475                                 pb = &dmap->d_dma->d_dbuf[j - 1];
 1476 
 1477                         pb->pb_addr = htole32(packp);
 1478 
 1479                         if (dtheend) {
 1480                                 if (packl > dtheend) {
 1481                                         pb->pb_len = htole32(dtheend);
 1482                                         dtheend = 0;
 1483                                 } else {
 1484                                         pb->pb_len = htole32(packl);
 1485                                         dtheend -= packl;
 1486                                 }
 1487                         } else
 1488                                 pb->pb_len = htole32(packl);
 1489 
 1490                         if ((i + 1) == q->q_dst_nsegs) {
 1491                                 if (maccrd)
 1492                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 1493                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 1494                                 else
 1495                                         pb->pb_next = 0;
 1496                         } else
 1497                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 1498                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
 1499                         j++;
 1500                 }
 1501         }
 1502 
 1503         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
 1504             offsetof(struct ubsec_dmachunk, d_ctx));
 1505 
 1506         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
 1507                 struct ubsec_pktctx_long *ctxl;
 1508 
 1509                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
 1510                     offsetof(struct ubsec_dmachunk, d_ctx));
 1511 
 1512                 /* transform small context into long context */
 1513                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
 1514                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
 1515                 ctxl->pc_flags = ctx.pc_flags;
 1516                 ctxl->pc_offset = ctx.pc_offset;
 1517                 for (i = 0; i < 6; i++)
 1518                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
 1519                 for (i = 0; i < 5; i++)
 1520                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
 1521                 for (i = 0; i < 5; i++)
 1522                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
 1523                 ctxl->pc_iv[0] = ctx.pc_iv[0];
 1524                 ctxl->pc_iv[1] = ctx.pc_iv[1];
 1525         } else
 1526                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
 1527                     offsetof(struct ubsec_dmachunk, d_ctx),
 1528                     sizeof(struct ubsec_pktctx));
 1529 
 1530         mtx_lock(&sc->sc_mcr1lock);
 1531         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
 1532         sc->sc_nqueue++;
 1533         ubsecstats.hst_ipackets++;
 1534         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
 1535         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
 1536                 ubsec_feed(sc);
 1537         mtx_unlock(&sc->sc_mcr1lock);
 1538         return (0);
 1539 
 1540 errout:
 1541         if (q != NULL) {
 1542                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
 1543                         m_freem(q->q_dst_m);
 1544 
 1545                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 1546                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 1547                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 1548                 }
 1549                 if (q->q_src_map != NULL) {
 1550                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 1551                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 1552                 }
 1553         }
 1554         if (q != NULL || err == ERESTART) {
 1555                 mtx_lock(&sc->sc_freeqlock);
 1556                 if (q != NULL)
 1557                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 1558                 if (err == ERESTART)
 1559                         sc->sc_needwakeup |= CRYPTO_SYMQ;
 1560                 mtx_unlock(&sc->sc_freeqlock);
 1561         }
 1562         if (err != ERESTART) {
 1563                 crp->crp_etype = err;
 1564                 crypto_done(crp);
 1565         }
 1566         return (err);
 1567 }
 1568 
 1569 static void
 1570 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
 1571 {
 1572         struct cryptop *crp = (struct cryptop *)q->q_crp;
 1573         struct cryptodesc *crd;
 1574         struct ubsec_dma *dmap = q->q_dma;
 1575 
 1576         ubsecstats.hst_opackets++;
 1577         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
 1578 
 1579         ubsec_dma_sync(&dmap->d_alloc,
 1580             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 1581         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 1582                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
 1583                     BUS_DMASYNC_POSTREAD);
 1584                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 1585                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 1586         }
 1587         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
 1588         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 1589         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 1590 
 1591         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
 1592                 m_freem(q->q_src_m);
 1593                 crp->crp_buf = (caddr_t)q->q_dst_m;
 1594         }
 1595 
 1596         /* copy out IV for future use */
 1597         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
 1598                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 1599                         if (crd->crd_alg != CRYPTO_DES_CBC &&
 1600                             crd->crd_alg != CRYPTO_3DES_CBC)
 1601                                 continue;
 1602                         crypto_copydata(crp->crp_flags, crp->crp_buf,
 1603                             crd->crd_skip + crd->crd_len - 8, 8,
 1604                             (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
 1605                         break;
 1606                 }
 1607         }
 1608 
 1609         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 1610                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
 1611                     crd->crd_alg != CRYPTO_SHA1_HMAC)
 1612                         continue;
 1613                 crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
 1614                     sc->sc_sessions[q->q_sesn].ses_mlen,
 1615                     (caddr_t)dmap->d_dma->d_macbuf);
 1616                 break;
 1617         }
 1618         mtx_lock(&sc->sc_freeqlock);
 1619         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 1620         mtx_unlock(&sc->sc_freeqlock);
 1621         crypto_done(crp);
 1622 }
 1623 
 1624 static void
 1625 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
 1626 {
 1627         int i, j, dlen, slen;
 1628         caddr_t dptr, sptr;
 1629 
 1630         j = 0;
 1631         sptr = srcm->m_data;
 1632         slen = srcm->m_len;
 1633         dptr = dstm->m_data;
 1634         dlen = dstm->m_len;
 1635 
 1636         while (1) {
 1637                 for (i = 0; i < min(slen, dlen); i++) {
 1638                         if (j < hoffset || j >= toffset)
 1639                                 *dptr++ = *sptr++;
 1640                         slen--;
 1641                         dlen--;
 1642                         j++;
 1643                 }
 1644                 if (slen == 0) {
 1645                         srcm = srcm->m_next;
 1646                         if (srcm == NULL)
 1647                                 return;
 1648                         sptr = srcm->m_data;
 1649                         slen = srcm->m_len;
 1650                 }
 1651                 if (dlen == 0) {
 1652                         dstm = dstm->m_next;
 1653                         if (dstm == NULL)
 1654                                 return;
 1655                         dptr = dstm->m_data;
 1656                         dlen = dstm->m_len;
 1657                 }
 1658         }
 1659 }
 1660 
 1661 /*
 1662  * feed the key generator, must be called at splimp() or higher.
 1663  */
 1664 static int
 1665 ubsec_feed2(struct ubsec_softc *sc)
 1666 {
 1667         struct ubsec_q2 *q;
 1668 
 1669         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
 1670                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
 1671                         break;
 1672                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
 1673 
 1674                 ubsec_dma_sync(&q->q_mcr,
 1675                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1676                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
 1677 
 1678                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
 1679                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
 1680                 --sc->sc_nqueue2;
 1681                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
 1682         }
 1683         return (0);
 1684 }
 1685 
 1686 /*
 1687  * Callback for handling random numbers
 1688  */
 1689 static void
 1690 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
 1691 {
 1692         struct cryptkop *krp;
 1693         struct ubsec_ctx_keyop *ctx;
 1694 
 1695         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
 1696         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
 1697 
 1698         switch (q->q_type) {
 1699 #ifndef UBSEC_NO_RNG
 1700         case UBS_CTXOP_RNGBYPASS: {
 1701                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
 1702 
 1703                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
 1704                 (*sc->sc_harvest)(sc->sc_rndtest,
 1705                         rng->rng_buf.dma_vaddr,
 1706                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
 1707                 rng->rng_used = 0;
 1708                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 1709                 break;
 1710         }
 1711 #endif
 1712         case UBS_CTXOP_MODEXP: {
 1713                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 1714                 u_int rlen, clen;
 1715 
 1716                 krp = me->me_krp;
 1717                 rlen = (me->me_modbits + 7) / 8;
 1718                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
 1719 
 1720                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
 1721                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
 1722                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
 1723                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
 1724 
 1725                 if (clen < rlen)
 1726                         krp->krp_status = E2BIG;
 1727                 else {
 1728                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
 1729                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
 1730                                     (krp->krp_param[krp->krp_iparams].crp_nbits
 1731                                         + 7) / 8);
 1732                                 bcopy(me->me_C.dma_vaddr,
 1733                                     krp->krp_param[krp->krp_iparams].crp_p,
 1734                                     (me->me_modbits + 7) / 8);
 1735                         } else
 1736                                 ubsec_kshift_l(me->me_shiftbits,
 1737                                     me->me_C.dma_vaddr, me->me_normbits,
 1738                                     krp->krp_param[krp->krp_iparams].crp_p,
 1739                                     krp->krp_param[krp->krp_iparams].crp_nbits);
 1740                 }
 1741 
 1742                 crypto_kdone(krp);
 1743 
 1744                 /* bzero all potentially sensitive data */
 1745                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 1746                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 1747                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 1748                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 1749 
 1750                 /* Can't free here, so put us on the free list. */
 1751                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
 1752                 break;
 1753         }
 1754         case UBS_CTXOP_RSAPRIV: {
 1755                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 1756                 u_int len;
 1757 
 1758                 krp = rp->rpr_krp;
 1759                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
 1760                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
 1761 
 1762                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
 1763                 bcopy(rp->rpr_msgout.dma_vaddr,
 1764                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
 1765 
 1766                 crypto_kdone(krp);
 1767 
 1768                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 1769                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 1770                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
 1771 
 1772                 /* Can't free here, so put us on the free list. */
 1773                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
 1774                 break;
 1775         }
 1776         default:
 1777                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
 1778                     letoh16(ctx->ctx_op));
 1779                 break;
 1780         }
 1781 }
 1782 
 1783 #ifndef UBSEC_NO_RNG
 1784 static void
 1785 ubsec_rng(void *vsc)
 1786 {
 1787         struct ubsec_softc *sc = vsc;
 1788         struct ubsec_q2_rng *rng = &sc->sc_rng;
 1789         struct ubsec_mcr *mcr;
 1790         struct ubsec_ctx_rngbypass *ctx;
 1791 
 1792         mtx_lock(&sc->sc_mcr2lock);
 1793         if (rng->rng_used) {
 1794                 mtx_unlock(&sc->sc_mcr2lock);
 1795                 return;
 1796         }
 1797         sc->sc_nqueue2++;
 1798         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
 1799                 goto out;
 1800 
 1801         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
 1802         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
 1803 
 1804         mcr->mcr_pkts = htole16(1);
 1805         mcr->mcr_flags = 0;
 1806         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
 1807         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
 1808         mcr->mcr_ipktbuf.pb_len = 0;
 1809         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
 1810         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
 1811         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
 1812             UBS_PKTBUF_LEN);
 1813         mcr->mcr_opktbuf.pb_next = 0;
 1814 
 1815         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
 1816         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
 1817         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
 1818 
 1819         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
 1820 
 1821         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
 1822         rng->rng_used = 1;
 1823         ubsec_feed2(sc);
 1824         ubsecstats.hst_rng++;
 1825         mtx_unlock(&sc->sc_mcr2lock);
 1826 
 1827         return;
 1828 
 1829 out:
 1830         /*
 1831          * Something weird happened, generate our own call back.
 1832          */
 1833         sc->sc_nqueue2--;
 1834         mtx_unlock(&sc->sc_mcr2lock);
 1835         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 1836 }
 1837 #endif /* UBSEC_NO_RNG */
 1838 
 1839 static void
 1840 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 1841 {
 1842         bus_addr_t *paddr = (bus_addr_t*) arg;
 1843         *paddr = segs->ds_addr;
 1844 }
 1845 
 1846 static int
 1847 ubsec_dma_malloc(
 1848         struct ubsec_softc *sc,
 1849         bus_size_t size,
 1850         struct ubsec_dma_alloc *dma,
 1851         int mapflags
 1852 )
 1853 {
 1854         int r;
 1855 
 1856         /* XXX could specify sc_dmat as parent but that just adds overhead */
 1857         r = bus_dma_tag_create(NULL,                    /* parent */
 1858                                1, 0,                    /* alignment, bounds */
 1859                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 1860                                BUS_SPACE_MAXADDR,       /* highaddr */
 1861                                NULL, NULL,              /* filter, filterarg */
 1862                                size,                    /* maxsize */
 1863                                1,                       /* nsegments */
 1864                                size,                    /* maxsegsize */
 1865                                BUS_DMA_ALLOCNOW,        /* flags */
 1866                                NULL, NULL,              /* lockfunc, lockarg */
 1867                                &dma->dma_tag);
 1868         if (r != 0) {
 1869                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 1870                         "bus_dma_tag_create failed; error %u\n", r);
 1871                 goto fail_0;
 1872         }
 1873 
 1874         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
 1875         if (r != 0) {
 1876                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 1877                         "bus_dmamap_create failed; error %u\n", r);
 1878                 goto fail_1;
 1879         }
 1880 
 1881         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 1882                              BUS_DMA_NOWAIT, &dma->dma_map);
 1883         if (r != 0) {
 1884                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 1885                         "bus_dmammem_alloc failed; size %zu, error %u\n",
 1886                         size, r);
 1887                 goto fail_2;
 1888         }
 1889 
 1890         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 1891                             size,
 1892                             ubsec_dmamap_cb,
 1893                             &dma->dma_paddr,
 1894                             mapflags | BUS_DMA_NOWAIT);
 1895         if (r != 0) {
 1896                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 1897                         "bus_dmamap_load failed; error %u\n", r);
 1898                 goto fail_3;
 1899         }
 1900 
 1901         dma->dma_size = size;
 1902         return (0);
 1903 
 1904 fail_3:
 1905         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 1906 fail_2:
 1907         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 1908 fail_1:
 1909         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 1910         bus_dma_tag_destroy(dma->dma_tag);
 1911 fail_0:
 1912         dma->dma_map = NULL;
 1913         dma->dma_tag = NULL;
 1914         return (r);
 1915 }
 1916 
 1917 static void
 1918 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
 1919 {
 1920         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 1921         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 1922         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 1923         bus_dma_tag_destroy(dma->dma_tag);
 1924 }
 1925 
 1926 /*
 1927  * Resets the board.  Values in the regesters are left as is
 1928  * from the reset (i.e. initial values are assigned elsewhere).
 1929  */
 1930 static void
 1931 ubsec_reset_board(struct ubsec_softc *sc)
 1932 {
 1933     volatile u_int32_t ctrl;
 1934 
 1935     ctrl = READ_REG(sc, BS_CTRL);
 1936     ctrl |= BS_CTRL_RESET;
 1937     WRITE_REG(sc, BS_CTRL, ctrl);
 1938 
 1939     /*
 1940      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
 1941      */
 1942     DELAY(10);
 1943 }
 1944 
 1945 /*
 1946  * Init Broadcom registers
 1947  */
 1948 static void
 1949 ubsec_init_board(struct ubsec_softc *sc)
 1950 {
 1951         u_int32_t ctrl;
 1952 
 1953         ctrl = READ_REG(sc, BS_CTRL);
 1954         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
 1955         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
 1956 
 1957         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
 1958                 ctrl |= BS_CTRL_MCR2INT;
 1959         else
 1960                 ctrl &= ~BS_CTRL_MCR2INT;
 1961 
 1962         if (sc->sc_flags & UBS_FLAGS_HWNORM)
 1963                 ctrl &= ~BS_CTRL_SWNORM;
 1964 
 1965         WRITE_REG(sc, BS_CTRL, ctrl);
 1966 }
 1967 
 1968 /*
 1969  * Init Broadcom PCI registers
 1970  */
 1971 static void
 1972 ubsec_init_pciregs(device_t dev)
 1973 {
 1974 #if 0
 1975         u_int32_t misc;
 1976 
 1977         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
 1978         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
 1979             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
 1980         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
 1981             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
 1982         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
 1983 #endif
 1984 
 1985         /*
 1986          * This will set the cache line size to 1, this will
 1987          * force the BCM58xx chip just to do burst read/writes.
 1988          * Cache line read/writes are to slow
 1989          */
 1990         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
 1991 }
 1992 
 1993 /*
 1994  * Clean up after a chip crash.
 1995  * It is assumed that the caller in splimp()
 1996  */
 1997 static void
 1998 ubsec_cleanchip(struct ubsec_softc *sc)
 1999 {
 2000         struct ubsec_q *q;
 2001 
 2002         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 2003                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
 2004                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 2005                 ubsec_free_q(sc, q);
 2006         }
 2007         sc->sc_nqchip = 0;
 2008 }
 2009 
 2010 /*
 2011  * free a ubsec_q
 2012  * It is assumed that the caller is within splimp().
 2013  */
 2014 static int
 2015 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
 2016 {
 2017         struct ubsec_q *q2;
 2018         struct cryptop *crp;
 2019         int npkts;
 2020         int i;
 2021 
 2022         npkts = q->q_nstacked_mcrs;
 2023 
 2024         for (i = 0; i < npkts; i++) {
 2025                 if(q->q_stacked_mcr[i]) {
 2026                         q2 = q->q_stacked_mcr[i];
 2027 
 2028                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
 2029                                 m_freem(q2->q_dst_m);
 2030 
 2031                         crp = (struct cryptop *)q2->q_crp;
 2032 
 2033                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
 2034 
 2035                         crp->crp_etype = EFAULT;
 2036                         crypto_done(crp);
 2037                 } else {
 2038                         break;
 2039                 }
 2040         }
 2041 
 2042         /*
 2043          * Free header MCR
 2044          */
 2045         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
 2046                 m_freem(q->q_dst_m);
 2047 
 2048         crp = (struct cryptop *)q->q_crp;
 2049 
 2050         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 2051 
 2052         crp->crp_etype = EFAULT;
 2053         crypto_done(crp);
 2054         return(0);
 2055 }
 2056 
 2057 /*
 2058  * Routine to reset the chip and clean up.
 2059  * It is assumed that the caller is in splimp()
 2060  */
 2061 static void
 2062 ubsec_totalreset(struct ubsec_softc *sc)
 2063 {
 2064         ubsec_reset_board(sc);
 2065         ubsec_init_board(sc);
 2066         ubsec_cleanchip(sc);
 2067 }
 2068 
 2069 static int
 2070 ubsec_dmamap_aligned(struct ubsec_operand *op)
 2071 {
 2072         int i;
 2073 
 2074         for (i = 0; i < op->nsegs; i++) {
 2075                 if (op->segs[i].ds_addr & 3)
 2076                         return (0);
 2077                 if ((i != (op->nsegs - 1)) &&
 2078                     (op->segs[i].ds_len & 3))
 2079                         return (0);
 2080         }
 2081         return (1);
 2082 }
 2083 
 2084 static void
 2085 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
 2086 {
 2087         switch (q->q_type) {
 2088         case UBS_CTXOP_MODEXP: {
 2089                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 2090 
 2091                 ubsec_dma_free(sc, &me->me_q.q_mcr);
 2092                 ubsec_dma_free(sc, &me->me_q.q_ctx);
 2093                 ubsec_dma_free(sc, &me->me_M);
 2094                 ubsec_dma_free(sc, &me->me_E);
 2095                 ubsec_dma_free(sc, &me->me_C);
 2096                 ubsec_dma_free(sc, &me->me_epb);
 2097                 free(me, M_DEVBUF);
 2098                 break;
 2099         }
 2100         case UBS_CTXOP_RSAPRIV: {
 2101                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 2102 
 2103                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 2104                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
 2105                 ubsec_dma_free(sc, &rp->rpr_msgin);
 2106                 ubsec_dma_free(sc, &rp->rpr_msgout);
 2107                 free(rp, M_DEVBUF);
 2108                 break;
 2109         }
 2110         default:
 2111                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
 2112                 break;
 2113         }
 2114 }
 2115 
 2116 static int
 2117 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
 2118 {
 2119         struct ubsec_softc *sc = device_get_softc(dev);
 2120         int r;
 2121 
 2122         if (krp == NULL || krp->krp_callback == NULL)
 2123                 return (EINVAL);
 2124 
 2125         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
 2126                 struct ubsec_q2 *q;
 2127 
 2128                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
 2129                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
 2130                 ubsec_kfree(sc, q);
 2131         }
 2132 
 2133         switch (krp->krp_op) {
 2134         case CRK_MOD_EXP:
 2135                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
 2136                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
 2137                 else
 2138                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
 2139                 break;
 2140         case CRK_MOD_EXP_CRT:
 2141                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
 2142         default:
 2143                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
 2144                     krp->krp_op);
 2145                 krp->krp_status = EOPNOTSUPP;
 2146                 crypto_kdone(krp);
 2147                 return (0);
 2148         }
 2149         return (0);                     /* silence compiler */
 2150 }
 2151 
 2152 /*
 2153  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
 2154  */
 2155 static int
 2156 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 2157 {
 2158         struct ubsec_q2_modexp *me;
 2159         struct ubsec_mcr *mcr;
 2160         struct ubsec_ctx_modexp *ctx;
 2161         struct ubsec_pktbuf *epb;
 2162         int err = 0;
 2163         u_int nbits, normbits, mbits, shiftbits, ebits;
 2164 
 2165         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 2166         if (me == NULL) {
 2167                 err = ENOMEM;
 2168                 goto errout;
 2169         }
 2170         bzero(me, sizeof *me);
 2171         me->me_krp = krp;
 2172         me->me_q.q_type = UBS_CTXOP_MODEXP;
 2173 
 2174         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 2175         if (nbits <= 512)
 2176                 normbits = 512;
 2177         else if (nbits <= 768)
 2178                 normbits = 768;
 2179         else if (nbits <= 1024)
 2180                 normbits = 1024;
 2181         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 2182                 normbits = 1536;
 2183         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 2184                 normbits = 2048;
 2185         else {
 2186                 err = E2BIG;
 2187                 goto errout;
 2188         }
 2189 
 2190         shiftbits = normbits - nbits;
 2191 
 2192         me->me_modbits = nbits;
 2193         me->me_shiftbits = shiftbits;
 2194         me->me_normbits = normbits;
 2195 
 2196         /* Sanity check: result bits must be >= true modulus bits. */
 2197         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 2198                 err = ERANGE;
 2199                 goto errout;
 2200         }
 2201 
 2202         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 2203             &me->me_q.q_mcr, 0)) {
 2204                 err = ENOMEM;
 2205                 goto errout;
 2206         }
 2207         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 2208 
 2209         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 2210             &me->me_q.q_ctx, 0)) {
 2211                 err = ENOMEM;
 2212                 goto errout;
 2213         }
 2214 
 2215         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 2216         if (mbits > nbits) {
 2217                 err = E2BIG;
 2218                 goto errout;
 2219         }
 2220         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 2221                 err = ENOMEM;
 2222                 goto errout;
 2223         }
 2224         ubsec_kshift_r(shiftbits,
 2225             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
 2226             me->me_M.dma_vaddr, normbits);
 2227 
 2228         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 2229                 err = ENOMEM;
 2230                 goto errout;
 2231         }
 2232         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 2233 
 2234         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 2235         if (ebits > nbits) {
 2236                 err = E2BIG;
 2237                 goto errout;
 2238         }
 2239         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 2240                 err = ENOMEM;
 2241                 goto errout;
 2242         }
 2243         ubsec_kshift_r(shiftbits,
 2244             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
 2245             me->me_E.dma_vaddr, normbits);
 2246 
 2247         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 2248             &me->me_epb, 0)) {
 2249                 err = ENOMEM;
 2250                 goto errout;
 2251         }
 2252         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 2253         epb->pb_addr = htole32(me->me_E.dma_paddr);
 2254         epb->pb_next = 0;
 2255         epb->pb_len = htole32(normbits / 8);
 2256 
 2257 #ifdef UBSEC_DEBUG
 2258         if (ubsec_debug) {
 2259                 printf("Epb ");
 2260                 ubsec_dump_pb(epb);
 2261         }
 2262 #endif
 2263 
 2264         mcr->mcr_pkts = htole16(1);
 2265         mcr->mcr_flags = 0;
 2266         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 2267         mcr->mcr_reserved = 0;
 2268         mcr->mcr_pktlen = 0;
 2269 
 2270         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 2271         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 2272         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 2273 
 2274         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 2275         mcr->mcr_opktbuf.pb_next = 0;
 2276         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 2277 
 2278 #ifdef DIAGNOSTIC
 2279         /* Misaligned output buffer will hang the chip. */
 2280         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 2281                 panic("%s: modexp invalid addr 0x%x\n",
 2282                     device_get_nameunit(sc->sc_dev),
 2283                     letoh32(mcr->mcr_opktbuf.pb_addr));
 2284         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 2285                 panic("%s: modexp invalid len 0x%x\n",
 2286                     device_get_nameunit(sc->sc_dev),
 2287                     letoh32(mcr->mcr_opktbuf.pb_len));
 2288 #endif
 2289 
 2290         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 2291         bzero(ctx, sizeof(*ctx));
 2292         ubsec_kshift_r(shiftbits,
 2293             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
 2294             ctx->me_N, normbits);
 2295         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 2296         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 2297         ctx->me_E_len = htole16(nbits);
 2298         ctx->me_N_len = htole16(nbits);
 2299 
 2300 #ifdef UBSEC_DEBUG
 2301         if (ubsec_debug) {
 2302                 ubsec_dump_mcr(mcr);
 2303                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 2304         }
 2305 #endif
 2306 
 2307         /*
 2308          * ubsec_feed2 will sync mcr and ctx, we just need to sync
 2309          * everything else.
 2310          */
 2311         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 2312         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 2313         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 2314         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 2315 
 2316         /* Enqueue and we're done... */
 2317         mtx_lock(&sc->sc_mcr2lock);
 2318         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 2319         ubsec_feed2(sc);
 2320         ubsecstats.hst_modexp++;
 2321         mtx_unlock(&sc->sc_mcr2lock);
 2322 
 2323         return (0);
 2324 
 2325 errout:
 2326         if (me != NULL) {
 2327                 if (me->me_q.q_mcr.dma_map != NULL)
 2328                         ubsec_dma_free(sc, &me->me_q.q_mcr);
 2329                 if (me->me_q.q_ctx.dma_map != NULL) {
 2330                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 2331                         ubsec_dma_free(sc, &me->me_q.q_ctx);
 2332                 }
 2333                 if (me->me_M.dma_map != NULL) {
 2334                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 2335                         ubsec_dma_free(sc, &me->me_M);
 2336                 }
 2337                 if (me->me_E.dma_map != NULL) {
 2338                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 2339                         ubsec_dma_free(sc, &me->me_E);
 2340                 }
 2341                 if (me->me_C.dma_map != NULL) {
 2342                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 2343                         ubsec_dma_free(sc, &me->me_C);
 2344                 }
 2345                 if (me->me_epb.dma_map != NULL)
 2346                         ubsec_dma_free(sc, &me->me_epb);
 2347                 free(me, M_DEVBUF);
 2348         }
 2349         krp->krp_status = err;
 2350         crypto_kdone(krp);
 2351         return (0);
 2352 }
 2353 
 2354 /*
 2355  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
 2356  */
 2357 static int
 2358 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 2359 {
 2360         struct ubsec_q2_modexp *me;
 2361         struct ubsec_mcr *mcr;
 2362         struct ubsec_ctx_modexp *ctx;
 2363         struct ubsec_pktbuf *epb;
 2364         int err = 0;
 2365         u_int nbits, normbits, mbits, shiftbits, ebits;
 2366 
 2367         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 2368         if (me == NULL) {
 2369                 err = ENOMEM;
 2370                 goto errout;
 2371         }
 2372         bzero(me, sizeof *me);
 2373         me->me_krp = krp;
 2374         me->me_q.q_type = UBS_CTXOP_MODEXP;
 2375 
 2376         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 2377         if (nbits <= 512)
 2378                 normbits = 512;
 2379         else if (nbits <= 768)
 2380                 normbits = 768;
 2381         else if (nbits <= 1024)
 2382                 normbits = 1024;
 2383         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 2384                 normbits = 1536;
 2385         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 2386                 normbits = 2048;
 2387         else {
 2388                 err = E2BIG;
 2389                 goto errout;
 2390         }
 2391 
 2392         shiftbits = normbits - nbits;
 2393 
 2394         /* XXX ??? */
 2395         me->me_modbits = nbits;
 2396         me->me_shiftbits = shiftbits;
 2397         me->me_normbits = normbits;
 2398 
 2399         /* Sanity check: result bits must be >= true modulus bits. */
 2400         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 2401                 err = ERANGE;
 2402                 goto errout;
 2403         }
 2404 
 2405         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 2406             &me->me_q.q_mcr, 0)) {
 2407                 err = ENOMEM;
 2408                 goto errout;
 2409         }
 2410         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 2411 
 2412         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 2413             &me->me_q.q_ctx, 0)) {
 2414                 err = ENOMEM;
 2415                 goto errout;
 2416         }
 2417 
 2418         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 2419         if (mbits > nbits) {
 2420                 err = E2BIG;
 2421                 goto errout;
 2422         }
 2423         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 2424                 err = ENOMEM;
 2425                 goto errout;
 2426         }
 2427         bzero(me->me_M.dma_vaddr, normbits / 8);
 2428         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
 2429             me->me_M.dma_vaddr, (mbits + 7) / 8);
 2430 
 2431         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 2432                 err = ENOMEM;
 2433                 goto errout;
 2434         }
 2435         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 2436 
 2437         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 2438         if (ebits > nbits) {
 2439                 err = E2BIG;
 2440                 goto errout;
 2441         }
 2442         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 2443                 err = ENOMEM;
 2444                 goto errout;
 2445         }
 2446         bzero(me->me_E.dma_vaddr, normbits / 8);
 2447         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
 2448             me->me_E.dma_vaddr, (ebits + 7) / 8);
 2449 
 2450         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 2451             &me->me_epb, 0)) {
 2452                 err = ENOMEM;
 2453                 goto errout;
 2454         }
 2455         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 2456         epb->pb_addr = htole32(me->me_E.dma_paddr);
 2457         epb->pb_next = 0;
 2458         epb->pb_len = htole32((ebits + 7) / 8);
 2459 
 2460 #ifdef UBSEC_DEBUG
 2461         if (ubsec_debug) {
 2462                 printf("Epb ");
 2463                 ubsec_dump_pb(epb);
 2464         }
 2465 #endif
 2466 
 2467         mcr->mcr_pkts = htole16(1);
 2468         mcr->mcr_flags = 0;
 2469         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 2470         mcr->mcr_reserved = 0;
 2471         mcr->mcr_pktlen = 0;
 2472 
 2473         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 2474         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 2475         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 2476 
 2477         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 2478         mcr->mcr_opktbuf.pb_next = 0;
 2479         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 2480 
 2481 #ifdef DIAGNOSTIC
 2482         /* Misaligned output buffer will hang the chip. */
 2483         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 2484                 panic("%s: modexp invalid addr 0x%x\n",
 2485                     device_get_nameunit(sc->sc_dev),
 2486                     letoh32(mcr->mcr_opktbuf.pb_addr));
 2487         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 2488                 panic("%s: modexp invalid len 0x%x\n",
 2489                     device_get_nameunit(sc->sc_dev),
 2490                     letoh32(mcr->mcr_opktbuf.pb_len));
 2491 #endif
 2492 
 2493         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 2494         bzero(ctx, sizeof(*ctx));
 2495         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
 2496             (nbits + 7) / 8);
 2497         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 2498         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 2499         ctx->me_E_len = htole16(ebits);
 2500         ctx->me_N_len = htole16(nbits);
 2501 
 2502 #ifdef UBSEC_DEBUG
 2503         if (ubsec_debug) {
 2504                 ubsec_dump_mcr(mcr);
 2505                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 2506         }
 2507 #endif
 2508 
 2509         /*
 2510          * ubsec_feed2 will sync mcr and ctx, we just need to sync
 2511          * everything else.
 2512          */
 2513         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 2514         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 2515         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 2516         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 2517 
 2518         /* Enqueue and we're done... */
 2519         mtx_lock(&sc->sc_mcr2lock);
 2520         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 2521         ubsec_feed2(sc);
 2522         mtx_unlock(&sc->sc_mcr2lock);
 2523 
 2524         return (0);
 2525 
 2526 errout:
 2527         if (me != NULL) {
 2528                 if (me->me_q.q_mcr.dma_map != NULL)
 2529                         ubsec_dma_free(sc, &me->me_q.q_mcr);
 2530                 if (me->me_q.q_ctx.dma_map != NULL) {
 2531                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 2532                         ubsec_dma_free(sc, &me->me_q.q_ctx);
 2533                 }
 2534                 if (me->me_M.dma_map != NULL) {
 2535                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 2536                         ubsec_dma_free(sc, &me->me_M);
 2537                 }
 2538                 if (me->me_E.dma_map != NULL) {
 2539                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 2540                         ubsec_dma_free(sc, &me->me_E);
 2541                 }
 2542                 if (me->me_C.dma_map != NULL) {
 2543                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 2544                         ubsec_dma_free(sc, &me->me_C);
 2545                 }
 2546                 if (me->me_epb.dma_map != NULL)
 2547                         ubsec_dma_free(sc, &me->me_epb);
 2548                 free(me, M_DEVBUF);
 2549         }
 2550         krp->krp_status = err;
 2551         crypto_kdone(krp);
 2552         return (0);
 2553 }
 2554 
 2555 static int
 2556 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 2557 {
 2558         struct ubsec_q2_rsapriv *rp = NULL;
 2559         struct ubsec_mcr *mcr;
 2560         struct ubsec_ctx_rsapriv *ctx;
 2561         int err = 0;
 2562         u_int padlen, msglen;
 2563 
 2564         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
 2565         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
 2566         if (msglen > padlen)
 2567                 padlen = msglen;
 2568 
 2569         if (padlen <= 256)
 2570                 padlen = 256;
 2571         else if (padlen <= 384)
 2572                 padlen = 384;
 2573         else if (padlen <= 512)
 2574                 padlen = 512;
 2575         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
 2576                 padlen = 768;
 2577         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
 2578                 padlen = 1024;
 2579         else {
 2580                 err = E2BIG;
 2581                 goto errout;
 2582         }
 2583 
 2584         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
 2585                 err = E2BIG;
 2586                 goto errout;
 2587         }
 2588 
 2589         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
 2590                 err = E2BIG;
 2591                 goto errout;
 2592         }
 2593 
 2594         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
 2595                 err = E2BIG;
 2596                 goto errout;
 2597         }
 2598 
 2599         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
 2600         if (rp == NULL)
 2601                 return (ENOMEM);
 2602         bzero(rp, sizeof *rp);
 2603         rp->rpr_krp = krp;
 2604         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
 2605 
 2606         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 2607             &rp->rpr_q.q_mcr, 0)) {
 2608                 err = ENOMEM;
 2609                 goto errout;
 2610         }
 2611         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
 2612 
 2613         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
 2614             &rp->rpr_q.q_ctx, 0)) {
 2615                 err = ENOMEM;
 2616                 goto errout;
 2617         }
 2618         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
 2619         bzero(ctx, sizeof *ctx);
 2620 
 2621         /* Copy in p */
 2622         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
 2623             &ctx->rpr_buf[0 * (padlen / 8)],
 2624             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
 2625 
 2626         /* Copy in q */
 2627         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
 2628             &ctx->rpr_buf[1 * (padlen / 8)],
 2629             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
 2630 
 2631         /* Copy in dp */
 2632         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
 2633             &ctx->rpr_buf[2 * (padlen / 8)],
 2634             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
 2635 
 2636         /* Copy in dq */
 2637         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
 2638             &ctx->rpr_buf[3 * (padlen / 8)],
 2639             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
 2640 
 2641         /* Copy in pinv */
 2642         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
 2643             &ctx->rpr_buf[4 * (padlen / 8)],
 2644             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
 2645 
 2646         msglen = padlen * 2;
 2647 
 2648         /* Copy in input message (aligned buffer/length). */
 2649         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
 2650                 /* Is this likely? */
 2651                 err = E2BIG;
 2652                 goto errout;
 2653         }
 2654         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
 2655                 err = ENOMEM;
 2656                 goto errout;
 2657         }
 2658         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
 2659         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
 2660             rp->rpr_msgin.dma_vaddr,
 2661             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
 2662 
 2663         /* Prepare space for output message (aligned buffer/length). */
 2664         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
 2665                 /* Is this likely? */
 2666                 err = E2BIG;
 2667                 goto errout;
 2668         }
 2669         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
 2670                 err = ENOMEM;
 2671                 goto errout;
 2672         }
 2673         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
 2674 
 2675         mcr->mcr_pkts = htole16(1);
 2676         mcr->mcr_flags = 0;
 2677         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
 2678         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
 2679         mcr->mcr_ipktbuf.pb_next = 0;
 2680         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
 2681         mcr->mcr_reserved = 0;
 2682         mcr->mcr_pktlen = htole16(msglen);
 2683         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
 2684         mcr->mcr_opktbuf.pb_next = 0;
 2685         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
 2686 
 2687 #ifdef DIAGNOSTIC
 2688         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
 2689                 panic("%s: rsapriv: invalid msgin %x(0x%jx)",
 2690                     device_get_nameunit(sc->sc_dev),
 2691                     rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
 2692         }
 2693         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
 2694                 panic("%s: rsapriv: invalid msgout %x(0x%jx)",
 2695                     device_get_nameunit(sc->sc_dev),
 2696                     rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
 2697         }
 2698 #endif
 2699 
 2700         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
 2701         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
 2702         ctx->rpr_q_len = htole16(padlen);
 2703         ctx->rpr_p_len = htole16(padlen);
 2704 
 2705         /*
 2706          * ubsec_feed2 will sync mcr and ctx, we just need to sync
 2707          * everything else.
 2708          */
 2709         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
 2710         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
 2711 
 2712         /* Enqueue and we're done... */
 2713         mtx_lock(&sc->sc_mcr2lock);
 2714         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
 2715         ubsec_feed2(sc);
 2716         ubsecstats.hst_modexpcrt++;
 2717         mtx_unlock(&sc->sc_mcr2lock);
 2718         return (0);
 2719 
 2720 errout:
 2721         if (rp != NULL) {
 2722                 if (rp->rpr_q.q_mcr.dma_map != NULL)
 2723                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 2724                 if (rp->rpr_msgin.dma_map != NULL) {
 2725                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 2726                         ubsec_dma_free(sc, &rp->rpr_msgin);
 2727                 }
 2728                 if (rp->rpr_msgout.dma_map != NULL) {
 2729                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 2730                         ubsec_dma_free(sc, &rp->rpr_msgout);
 2731                 }
 2732                 free(rp, M_DEVBUF);
 2733         }
 2734         krp->krp_status = err;
 2735         crypto_kdone(krp);
 2736         return (0);
 2737 }
 2738 
 2739 #ifdef UBSEC_DEBUG
 2740 static void
 2741 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
 2742 {
 2743         printf("addr 0x%x (0x%x) next 0x%x\n",
 2744             pb->pb_addr, pb->pb_len, pb->pb_next);
 2745 }
 2746 
 2747 static void
 2748 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
 2749 {
 2750         printf("CTX (0x%x):\n", c->ctx_len);
 2751         switch (letoh16(c->ctx_op)) {
 2752         case UBS_CTXOP_RNGBYPASS:
 2753         case UBS_CTXOP_RNGSHA1:
 2754                 break;
 2755         case UBS_CTXOP_MODEXP:
 2756         {
 2757                 struct ubsec_ctx_modexp *cx = (void *)c;
 2758                 int i, len;
 2759 
 2760                 printf(" Elen %u, Nlen %u\n",
 2761                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
 2762                 len = (cx->me_N_len + 7)/8;
 2763                 for (i = 0; i < len; i++)
 2764                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
 2765                 printf("\n");
 2766                 break;
 2767         }
 2768         default:
 2769                 printf("unknown context: %x\n", c->ctx_op);
 2770         }
 2771         printf("END CTX\n");
 2772 }
 2773 
 2774 static void
 2775 ubsec_dump_mcr(struct ubsec_mcr *mcr)
 2776 {
 2777         volatile struct ubsec_mcr_add *ma;
 2778         int i;
 2779 
 2780         printf("MCR:\n");
 2781         printf(" pkts: %u, flags 0x%x\n",
 2782             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
 2783         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
 2784         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
 2785                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
 2786                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
 2787                     letoh16(ma->mcr_reserved));
 2788                 printf(" %d: ipkt ", i);
 2789                 ubsec_dump_pb(&ma->mcr_ipktbuf);
 2790                 printf(" %d: opkt ", i);
 2791                 ubsec_dump_pb(&ma->mcr_opktbuf);
 2792                 ma++;
 2793         }
 2794         printf("END MCR\n");
 2795 }
 2796 #endif /* UBSEC_DEBUG */
 2797 
 2798 /*
 2799  * Return the number of significant bits of a big number.
 2800  */
 2801 static int
 2802 ubsec_ksigbits(struct crparam *cr)
 2803 {
 2804         u_int plen = (cr->crp_nbits + 7) / 8;
 2805         int i, sig = plen * 8;
 2806         u_int8_t c, *p = cr->crp_p;
 2807 
 2808         for (i = plen - 1; i >= 0; i--) {
 2809                 c = p[i];
 2810                 if (c != 0) {
 2811                         while ((c & 0x80) == 0) {
 2812                                 sig--;
 2813                                 c <<= 1;
 2814                         }
 2815                         break;
 2816                 }
 2817                 sig -= 8;
 2818         }
 2819         return (sig);
 2820 }
 2821 
 2822 static void
 2823 ubsec_kshift_r(
 2824         u_int shiftbits,
 2825         u_int8_t *src, u_int srcbits,
 2826         u_int8_t *dst, u_int dstbits)
 2827 {
 2828         u_int slen, dlen;
 2829         int i, si, di, n;
 2830 
 2831         slen = (srcbits + 7) / 8;
 2832         dlen = (dstbits + 7) / 8;
 2833 
 2834         for (i = 0; i < slen; i++)
 2835                 dst[i] = src[i];
 2836         for (i = 0; i < dlen - slen; i++)
 2837                 dst[slen + i] = 0;
 2838 
 2839         n = shiftbits / 8;
 2840         if (n != 0) {
 2841                 si = dlen - n - 1;
 2842                 di = dlen - 1;
 2843                 while (si >= 0)
 2844                         dst[di--] = dst[si--];
 2845                 while (di >= 0)
 2846                         dst[di--] = 0;
 2847         }
 2848 
 2849         n = shiftbits % 8;
 2850         if (n != 0) {
 2851                 for (i = dlen - 1; i > 0; i--)
 2852                         dst[i] = (dst[i] << n) |
 2853                             (dst[i - 1] >> (8 - n));
 2854                 dst[0] = dst[0] << n;
 2855         }
 2856 }
 2857 
 2858 static void
 2859 ubsec_kshift_l(
 2860         u_int shiftbits,
 2861         u_int8_t *src, u_int srcbits,
 2862         u_int8_t *dst, u_int dstbits)
 2863 {
 2864         int slen, dlen, i, n;
 2865 
 2866         slen = (srcbits + 7) / 8;
 2867         dlen = (dstbits + 7) / 8;
 2868 
 2869         n = shiftbits / 8;
 2870         for (i = 0; i < slen; i++)
 2871                 dst[i] = src[i + n];
 2872         for (i = 0; i < dlen - slen; i++)
 2873                 dst[slen + i] = 0;
 2874 
 2875         n = shiftbits % 8;
 2876         if (n != 0) {
 2877                 for (i = 0; i < (dlen - 1); i++)
 2878                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
 2879                 dst[dlen - 1] = dst[dlen - 1] >> n;
 2880         }
 2881 }

Cache object: dab6a8ae1d2bf7eba8adb2eac7f83ac8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.