1 /* $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $ */
2
3 /*
4 * Invertex AEON / Hifn 7751 driver
5 * Copyright (c) 1999 Invertex Inc. All rights reserved.
6 * Copyright (c) 1999 Theo de Raadt
7 * Copyright (c) 2000-2001 Network Security Technologies, Inc.
8 * http://www.netsec.net
9 * Copyright (c) 2003 Hifn Inc.
10 *
11 * This driver is based on a previous driver by Invertex, for which they
12 * requested: Please send any comments, feedback, bug-fixes, or feature
13 * requests to software@invertex.com.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 *
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. The name of the author may not be used to endorse or promote products
25 * derived from this software without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
28 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
29 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
30 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
31 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
32 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
36 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 *
38 * Effort sponsored in part by the Defense Advanced Research Projects
39 * Agency (DARPA) and Air Force Research Laboratory, Air Force
40 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
41 */
42
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD: releng/5.3/sys/dev/hifn/hifn7751.c 131575 2004-07-04 16:11:03Z stefanf $");
45
46 /*
47 * Driver for various Hifn encryption processors.
48 */
49 #include "opt_hifn.h"
50
51 #include <sys/param.h>
52 #include <sys/systm.h>
53 #include <sys/proc.h>
54 #include <sys/errno.h>
55 #include <sys/malloc.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/mbuf.h>
59 #include <sys/lock.h>
60 #include <sys/mutex.h>
61 #include <sys/sysctl.h>
62
63 #include <vm/vm.h>
64 #include <vm/pmap.h>
65
66 #include <machine/clock.h>
67 #include <machine/bus.h>
68 #include <machine/resource.h>
69 #include <sys/bus.h>
70 #include <sys/rman.h>
71
72 #include <opencrypto/cryptodev.h>
73 #include <sys/random.h>
74
75 #include <dev/pci/pcivar.h>
76 #include <dev/pci/pcireg.h>
77
78 #ifdef HIFN_RNDTEST
79 #include <dev/rndtest/rndtest.h>
80 #endif
81 #include <dev/hifn/hifn7751reg.h>
82 #include <dev/hifn/hifn7751var.h>
83
84 /*
85 * Prototypes and count for the pci_device structure
86 */
87 static int hifn_probe(device_t);
88 static int hifn_attach(device_t);
89 static int hifn_detach(device_t);
90 static int hifn_suspend(device_t);
91 static int hifn_resume(device_t);
92 static void hifn_shutdown(device_t);
93
94 static device_method_t hifn_methods[] = {
95 /* Device interface */
96 DEVMETHOD(device_probe, hifn_probe),
97 DEVMETHOD(device_attach, hifn_attach),
98 DEVMETHOD(device_detach, hifn_detach),
99 DEVMETHOD(device_suspend, hifn_suspend),
100 DEVMETHOD(device_resume, hifn_resume),
101 DEVMETHOD(device_shutdown, hifn_shutdown),
102
103 /* bus interface */
104 DEVMETHOD(bus_print_child, bus_generic_print_child),
105 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
106
107 { 0, 0 }
108 };
109 static driver_t hifn_driver = {
110 "hifn",
111 hifn_methods,
112 sizeof (struct hifn_softc)
113 };
114 static devclass_t hifn_devclass;
115
116 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
117 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
118 #ifdef HIFN_RNDTEST
119 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
120 #endif
121
122 static void hifn_reset_board(struct hifn_softc *, int);
123 static void hifn_reset_puc(struct hifn_softc *);
124 static void hifn_puc_wait(struct hifn_softc *);
125 static int hifn_enable_crypto(struct hifn_softc *);
126 static void hifn_set_retry(struct hifn_softc *sc);
127 static void hifn_init_dma(struct hifn_softc *);
128 static void hifn_init_pci_registers(struct hifn_softc *);
129 static int hifn_sramsize(struct hifn_softc *);
130 static int hifn_dramsize(struct hifn_softc *);
131 static int hifn_ramtype(struct hifn_softc *);
132 static void hifn_sessions(struct hifn_softc *);
133 static void hifn_intr(void *);
134 static u_int hifn_write_command(struct hifn_command *, u_int8_t *);
135 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
136 static int hifn_newsession(void *, u_int32_t *, struct cryptoini *);
137 static int hifn_freesession(void *, u_int64_t);
138 static int hifn_process(void *, struct cryptop *, int);
139 static void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
140 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
141 static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
142 static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
143 static int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
144 static int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
145 static int hifn_init_pubrng(struct hifn_softc *);
146 static void hifn_rng(void *);
147 static void hifn_tick(void *);
148 static void hifn_abort(struct hifn_softc *);
149 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
150
151 static void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
152 static void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
153
154 static __inline u_int32_t
155 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
156 {
157 u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
158 sc->sc_bar0_lastreg = (bus_size_t) -1;
159 return (v);
160 }
161 #define WRITE_REG_0(sc, reg, val) hifn_write_reg_0(sc, reg, val)
162
163 static __inline u_int32_t
164 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
165 {
166 u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
167 sc->sc_bar1_lastreg = (bus_size_t) -1;
168 return (v);
169 }
170 #define WRITE_REG_1(sc, reg, val) hifn_write_reg_1(sc, reg, val)
171
172 SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0, "Hifn driver parameters");
173
174 #ifdef HIFN_DEBUG
175 static int hifn_debug = 0;
176 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
177 0, "control debugging msgs");
178 #endif
179
180 static struct hifn_stats hifnstats;
181 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
182 hifn_stats, "driver statistics");
183 static int hifn_maxbatch = 1;
184 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
185 0, "max ops to batch w/o interrupt");
186
187 /*
188 * Probe for a supported device. The PCI vendor and device
189 * IDs are used to detect devices we know how to handle.
190 */
191 static int
192 hifn_probe(device_t dev)
193 {
194 if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
195 pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
196 return (0);
197 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
198 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
199 pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
200 pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
201 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
202 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
203 return (0);
204 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
205 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
206 return (0);
207 return (ENXIO);
208 }
209
210 static void
211 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
212 {
213 bus_addr_t *paddr = (bus_addr_t*) arg;
214 *paddr = segs->ds_addr;
215 }
216
217 static const char*
218 hifn_partname(struct hifn_softc *sc)
219 {
220 /* XXX sprintf numbers when not decoded */
221 switch (pci_get_vendor(sc->sc_dev)) {
222 case PCI_VENDOR_HIFN:
223 switch (pci_get_device(sc->sc_dev)) {
224 case PCI_PRODUCT_HIFN_6500: return "Hifn 6500";
225 case PCI_PRODUCT_HIFN_7751: return "Hifn 7751";
226 case PCI_PRODUCT_HIFN_7811: return "Hifn 7811";
227 case PCI_PRODUCT_HIFN_7951: return "Hifn 7951";
228 case PCI_PRODUCT_HIFN_7955: return "Hifn 7955";
229 case PCI_PRODUCT_HIFN_7956: return "Hifn 7956";
230 }
231 return "Hifn unknown-part";
232 case PCI_VENDOR_INVERTEX:
233 switch (pci_get_device(sc->sc_dev)) {
234 case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
235 }
236 return "Invertex unknown-part";
237 case PCI_VENDOR_NETSEC:
238 switch (pci_get_device(sc->sc_dev)) {
239 case PCI_PRODUCT_NETSEC_7751: return "NetSec 7751";
240 }
241 return "NetSec unknown-part";
242 }
243 return "Unknown-vendor unknown-part";
244 }
245
246 static void
247 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
248 {
249 random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
250 }
251
252 /*
253 * Attach an interface that successfully probed.
254 */
255 static int
256 hifn_attach(device_t dev)
257 {
258 struct hifn_softc *sc = device_get_softc(dev);
259 u_int32_t cmd;
260 caddr_t kva;
261 int rseg, rid;
262 char rbase;
263 u_int16_t ena, rev;
264
265 KASSERT(sc != NULL, ("hifn_attach: null software carrier!"));
266 bzero(sc, sizeof (*sc));
267 sc->sc_dev = dev;
268
269 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
270
271 /* XXX handle power management */
272
273 /*
274 * The 7951 and 795x have a random number generator and
275 * public key support; note this.
276 */
277 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
278 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
279 pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
280 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
281 sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
282 /*
283 * The 7811 has a random number generator and
284 * we also note it's identity 'cuz of some quirks.
285 */
286 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
287 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
288 sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
289
290 /*
291 * The 795x parts support AES.
292 */
293 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
294 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
295 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
296 sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
297
298 /*
299 * Configure support for memory-mapped access to
300 * registers and for DMA operations.
301 */
302 #define PCIM_ENA (PCIM_CMD_MEMEN|PCIM_CMD_BUSMASTEREN)
303 cmd = pci_read_config(dev, PCIR_COMMAND, 4);
304 cmd |= PCIM_ENA;
305 pci_write_config(dev, PCIR_COMMAND, cmd, 4);
306 cmd = pci_read_config(dev, PCIR_COMMAND, 4);
307 if ((cmd & PCIM_ENA) != PCIM_ENA) {
308 device_printf(dev, "failed to enable %s\n",
309 (cmd & PCIM_ENA) == 0 ?
310 "memory mapping & bus mastering" :
311 (cmd & PCIM_CMD_MEMEN) == 0 ?
312 "memory mapping" : "bus mastering");
313 goto fail_pci;
314 }
315 #undef PCIM_ENA
316
317 /*
318 * Setup PCI resources. Note that we record the bus
319 * tag and handle for each register mapping, this is
320 * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
321 * and WRITE_REG_1 macros throughout the driver.
322 */
323 rid = HIFN_BAR0;
324 sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
325 RF_ACTIVE);
326 if (sc->sc_bar0res == NULL) {
327 device_printf(dev, "cannot map bar%d register space\n", 0);
328 goto fail_pci;
329 }
330 sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
331 sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
332 sc->sc_bar0_lastreg = (bus_size_t) -1;
333
334 rid = HIFN_BAR1;
335 sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
336 RF_ACTIVE);
337 if (sc->sc_bar1res == NULL) {
338 device_printf(dev, "cannot map bar%d register space\n", 1);
339 goto fail_io0;
340 }
341 sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
342 sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
343 sc->sc_bar1_lastreg = (bus_size_t) -1;
344
345 hifn_set_retry(sc);
346
347 /*
348 * Setup the area where the Hifn DMA's descriptors
349 * and associated data structures.
350 */
351 if (bus_dma_tag_create(NULL, /* parent */
352 1, 0, /* alignment,boundary */
353 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
354 BUS_SPACE_MAXADDR, /* highaddr */
355 NULL, NULL, /* filter, filterarg */
356 HIFN_MAX_DMALEN, /* maxsize */
357 MAX_SCATTER, /* nsegments */
358 HIFN_MAX_SEGLEN, /* maxsegsize */
359 BUS_DMA_ALLOCNOW, /* flags */
360 NULL, /* lockfunc */
361 NULL, /* lockarg */
362 &sc->sc_dmat)) {
363 device_printf(dev, "cannot allocate DMA tag\n");
364 goto fail_io1;
365 }
366 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
367 device_printf(dev, "cannot create dma map\n");
368 bus_dma_tag_destroy(sc->sc_dmat);
369 goto fail_io1;
370 }
371 if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
372 device_printf(dev, "cannot alloc dma buffer\n");
373 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
374 bus_dma_tag_destroy(sc->sc_dmat);
375 goto fail_io1;
376 }
377 if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
378 sizeof (*sc->sc_dma),
379 hifn_dmamap_cb, &sc->sc_dma_physaddr,
380 BUS_DMA_NOWAIT)) {
381 device_printf(dev, "cannot load dma map\n");
382 bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
383 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
384 bus_dma_tag_destroy(sc->sc_dmat);
385 goto fail_io1;
386 }
387 sc->sc_dma = (struct hifn_dma *)kva;
388 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
389
390 KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
391 KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
392 KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
393 KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
394
395 /*
396 * Reset the board and do the ``secret handshake''
397 * to enable the crypto support. Then complete the
398 * initialization procedure by setting up the interrupt
399 * and hooking in to the system crypto support so we'll
400 * get used for system services like the crypto device,
401 * IPsec, RNG device, etc.
402 */
403 hifn_reset_board(sc, 0);
404
405 if (hifn_enable_crypto(sc) != 0) {
406 device_printf(dev, "crypto enabling failed\n");
407 goto fail_mem;
408 }
409 hifn_reset_puc(sc);
410
411 hifn_init_dma(sc);
412 hifn_init_pci_registers(sc);
413
414 /* XXX can't dynamically determine ram type for 795x; force dram */
415 if (sc->sc_flags & HIFN_IS_7956)
416 sc->sc_drammodel = 1;
417 else if (hifn_ramtype(sc))
418 goto fail_mem;
419
420 if (sc->sc_drammodel == 0)
421 hifn_sramsize(sc);
422 else
423 hifn_dramsize(sc);
424
425 /*
426 * Workaround for NetSec 7751 rev A: half ram size because two
427 * of the address lines were left floating
428 */
429 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
430 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
431 pci_get_revid(dev) == 0x61) /*XXX???*/
432 sc->sc_ramsize >>= 1;
433
434 /*
435 * Arrange the interrupt line.
436 */
437 rid = 0;
438 sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
439 RF_SHAREABLE|RF_ACTIVE);
440 if (sc->sc_irq == NULL) {
441 device_printf(dev, "could not map interrupt\n");
442 goto fail_mem;
443 }
444 /*
445 * NB: Network code assumes we are blocked with splimp()
446 * so make sure the IRQ is marked appropriately.
447 */
448 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
449 hifn_intr, sc, &sc->sc_intrhand)) {
450 device_printf(dev, "could not setup interrupt\n");
451 goto fail_intr2;
452 }
453
454 hifn_sessions(sc);
455
456 /*
457 * NB: Keep only the low 16 bits; this masks the chip id
458 * from the 7951.
459 */
460 rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
461
462 rseg = sc->sc_ramsize / 1024;
463 rbase = 'K';
464 if (sc->sc_ramsize >= (1024 * 1024)) {
465 rbase = 'M';
466 rseg /= 1024;
467 }
468 device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram, %u sessions\n",
469 hifn_partname(sc), rev,
470 rseg, rbase, sc->sc_drammodel ? 'd' : 's',
471 sc->sc_maxses);
472
473 sc->sc_cid = crypto_get_driverid(0);
474 if (sc->sc_cid < 0) {
475 device_printf(dev, "could not get crypto driver id\n");
476 goto fail_intr;
477 }
478
479 WRITE_REG_0(sc, HIFN_0_PUCNFG,
480 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
481 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
482
483 switch (ena) {
484 case HIFN_PUSTAT_ENA_2:
485 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
486 hifn_newsession, hifn_freesession, hifn_process, sc);
487 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0,
488 hifn_newsession, hifn_freesession, hifn_process, sc);
489 if (sc->sc_flags & HIFN_HAS_AES)
490 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
491 hifn_newsession, hifn_freesession,
492 hifn_process, sc);
493 /*FALLTHROUGH*/
494 case HIFN_PUSTAT_ENA_1:
495 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
496 hifn_newsession, hifn_freesession, hifn_process, sc);
497 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
498 hifn_newsession, hifn_freesession, hifn_process, sc);
499 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
500 hifn_newsession, hifn_freesession, hifn_process, sc);
501 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
502 hifn_newsession, hifn_freesession, hifn_process, sc);
503 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
504 hifn_newsession, hifn_freesession, hifn_process, sc);
505 break;
506 }
507
508 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
509 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
510
511 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
512 hifn_init_pubrng(sc);
513
514 callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
515 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
516
517 return (0);
518
519 fail_intr:
520 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
521 fail_intr2:
522 /* XXX don't store rid */
523 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
524 fail_mem:
525 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
526 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
527 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
528 bus_dma_tag_destroy(sc->sc_dmat);
529
530 /* Turn off DMA polling */
531 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
532 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
533 fail_io1:
534 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
535 fail_io0:
536 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
537 fail_pci:
538 mtx_destroy(&sc->sc_mtx);
539 return (ENXIO);
540 }
541
542 /*
543 * Detach an interface that successfully probed.
544 */
545 static int
546 hifn_detach(device_t dev)
547 {
548 struct hifn_softc *sc = device_get_softc(dev);
549
550 KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
551
552 /* disable interrupts */
553 WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
554
555 /*XXX other resources */
556 callout_stop(&sc->sc_tickto);
557 callout_stop(&sc->sc_rngto);
558 #ifdef HIFN_RNDTEST
559 if (sc->sc_rndtest)
560 rndtest_detach(sc->sc_rndtest);
561 #endif
562
563 /* Turn off DMA polling */
564 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
565 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
566
567 crypto_unregister_all(sc->sc_cid);
568
569 bus_generic_detach(dev); /*XXX should be no children, right? */
570
571 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
572 /* XXX don't store rid */
573 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
574
575 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
576 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
577 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
578 bus_dma_tag_destroy(sc->sc_dmat);
579
580 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
581 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
582
583 mtx_destroy(&sc->sc_mtx);
584
585 return (0);
586 }
587
588 /*
589 * Stop all chip I/O so that the kernel's probe routines don't
590 * get confused by errant DMAs when rebooting.
591 */
592 static void
593 hifn_shutdown(device_t dev)
594 {
595 #ifdef notyet
596 hifn_stop(device_get_softc(dev));
597 #endif
598 }
599
600 /*
601 * Device suspend routine. Stop the interface and save some PCI
602 * settings in case the BIOS doesn't restore them properly on
603 * resume.
604 */
605 static int
606 hifn_suspend(device_t dev)
607 {
608 struct hifn_softc *sc = device_get_softc(dev);
609 #ifdef notyet
610 int i;
611
612 hifn_stop(sc);
613 for (i = 0; i < 5; i++)
614 sc->saved_maps[i] = pci_read_config(dev, PCIR_BAR(i), 4);
615 sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4);
616 sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1);
617 sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
618 sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
619 #endif
620 sc->sc_suspended = 1;
621
622 return (0);
623 }
624
625 /*
626 * Device resume routine. Restore some PCI settings in case the BIOS
627 * doesn't, re-enable busmastering, and restart the interface if
628 * appropriate.
629 */
630 static int
631 hifn_resume(device_t dev)
632 {
633 struct hifn_softc *sc = device_get_softc(dev);
634 #ifdef notyet
635 int i;
636
637 /* better way to do this? */
638 for (i = 0; i < 5; i++)
639 pci_write_config(dev, PCIR_BAR(i), sc->saved_maps[i], 4);
640 pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4);
641 pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1);
642 pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1);
643 pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1);
644
645 /* reenable busmastering */
646 pci_enable_busmaster(dev);
647 pci_enable_io(dev, HIFN_RES);
648
649 /* reinitialize interface if necessary */
650 if (ifp->if_flags & IFF_UP)
651 rl_init(sc);
652 #endif
653 sc->sc_suspended = 0;
654
655 return (0);
656 }
657
658 static int
659 hifn_init_pubrng(struct hifn_softc *sc)
660 {
661 u_int32_t r;
662 int i;
663
664 #ifdef HIFN_RNDTEST
665 sc->sc_rndtest = rndtest_attach(sc->sc_dev);
666 if (sc->sc_rndtest)
667 sc->sc_harvest = rndtest_harvest;
668 else
669 sc->sc_harvest = default_harvest;
670 #else
671 sc->sc_harvest = default_harvest;
672 #endif
673 if ((sc->sc_flags & HIFN_IS_7811) == 0) {
674 /* Reset 7951 public key/rng engine */
675 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
676 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
677
678 for (i = 0; i < 100; i++) {
679 DELAY(1000);
680 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
681 HIFN_PUBRST_RESET) == 0)
682 break;
683 }
684
685 if (i == 100) {
686 device_printf(sc->sc_dev, "public key init failed\n");
687 return (1);
688 }
689 }
690
691 /* Enable the rng, if available */
692 if (sc->sc_flags & HIFN_HAS_RNG) {
693 if (sc->sc_flags & HIFN_IS_7811) {
694 r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
695 if (r & HIFN_7811_RNGENA_ENA) {
696 r &= ~HIFN_7811_RNGENA_ENA;
697 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
698 }
699 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
700 HIFN_7811_RNGCFG_DEFL);
701 r |= HIFN_7811_RNGENA_ENA;
702 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
703 } else
704 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
705 READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
706 HIFN_RNGCFG_ENA);
707
708 sc->sc_rngfirst = 1;
709 if (hz >= 100)
710 sc->sc_rnghz = hz / 100;
711 else
712 sc->sc_rnghz = 1;
713 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
714 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
715 }
716
717 /* Enable public key engine, if available */
718 if (sc->sc_flags & HIFN_HAS_PUBLIC) {
719 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
720 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
721 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
722 }
723
724 return (0);
725 }
726
727 static void
728 hifn_rng(void *vsc)
729 {
730 #define RANDOM_BITS(n) (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
731 struct hifn_softc *sc = vsc;
732 u_int32_t sts, num[2];
733 int i;
734
735 if (sc->sc_flags & HIFN_IS_7811) {
736 for (i = 0; i < 5; i++) {
737 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
738 if (sts & HIFN_7811_RNGSTS_UFL) {
739 device_printf(sc->sc_dev,
740 "RNG underflow: disabling\n");
741 return;
742 }
743 if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
744 break;
745
746 /*
747 * There are at least two words in the RNG FIFO
748 * at this point.
749 */
750 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
751 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
752 /* NB: discard first data read */
753 if (sc->sc_rngfirst)
754 sc->sc_rngfirst = 0;
755 else
756 (*sc->sc_harvest)(sc->sc_rndtest,
757 num, sizeof (num));
758 }
759 } else {
760 num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
761
762 /* NB: discard first data read */
763 if (sc->sc_rngfirst)
764 sc->sc_rngfirst = 0;
765 else
766 (*sc->sc_harvest)(sc->sc_rndtest,
767 num, sizeof (num[0]));
768 }
769
770 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
771 #undef RANDOM_BITS
772 }
773
774 static void
775 hifn_puc_wait(struct hifn_softc *sc)
776 {
777 int i;
778
779 for (i = 5000; i > 0; i--) {
780 DELAY(1);
781 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET))
782 break;
783 }
784 if (!i)
785 device_printf(sc->sc_dev, "proc unit did not reset\n");
786 }
787
788 /*
789 * Reset the processing unit.
790 */
791 static void
792 hifn_reset_puc(struct hifn_softc *sc)
793 {
794 /* Reset processing unit */
795 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
796 hifn_puc_wait(sc);
797 }
798
799 /*
800 * Set the Retry and TRDY registers; note that we set them to
801 * zero because the 7811 locks up when forced to retry (section
802 * 3.6 of "Specification Update SU-0014-04". Not clear if we
803 * should do this for all Hifn parts, but it doesn't seem to hurt.
804 */
805 static void
806 hifn_set_retry(struct hifn_softc *sc)
807 {
808 /* NB: RETRY only responds to 8-bit reads/writes */
809 pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
810 pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 4);
811 }
812
813 /*
814 * Resets the board. Values in the regesters are left as is
815 * from the reset (i.e. initial values are assigned elsewhere).
816 */
817 static void
818 hifn_reset_board(struct hifn_softc *sc, int full)
819 {
820 u_int32_t reg;
821
822 /*
823 * Set polling in the DMA configuration register to zero. 0x7 avoids
824 * resetting the board and zeros out the other fields.
825 */
826 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
827 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
828
829 /*
830 * Now that polling has been disabled, we have to wait 1 ms
831 * before resetting the board.
832 */
833 DELAY(1000);
834
835 /* Reset the DMA unit */
836 if (full) {
837 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
838 DELAY(1000);
839 } else {
840 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
841 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
842 hifn_reset_puc(sc);
843 }
844
845 KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
846 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
847
848 /* Bring dma unit out of reset */
849 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
850 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
851
852 hifn_puc_wait(sc);
853 hifn_set_retry(sc);
854
855 if (sc->sc_flags & HIFN_IS_7811) {
856 for (reg = 0; reg < 1000; reg++) {
857 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
858 HIFN_MIPSRST_CRAMINIT)
859 break;
860 DELAY(1000);
861 }
862 if (reg == 1000)
863 printf(": cram init timeout\n");
864 }
865 }
866
867 static u_int32_t
868 hifn_next_signature(u_int32_t a, u_int cnt)
869 {
870 int i;
871 u_int32_t v;
872
873 for (i = 0; i < cnt; i++) {
874
875 /* get the parity */
876 v = a & 0x80080125;
877 v ^= v >> 16;
878 v ^= v >> 8;
879 v ^= v >> 4;
880 v ^= v >> 2;
881 v ^= v >> 1;
882
883 a = (v & 1) ^ (a << 1);
884 }
885
886 return a;
887 }
888
889 struct pci2id {
890 u_short pci_vendor;
891 u_short pci_prod;
892 char card_id[13];
893 };
894 static struct pci2id pci2id[] = {
895 {
896 PCI_VENDOR_HIFN,
897 PCI_PRODUCT_HIFN_7951,
898 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
899 0x00, 0x00, 0x00, 0x00, 0x00 }
900 }, {
901 PCI_VENDOR_HIFN,
902 PCI_PRODUCT_HIFN_7955,
903 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
904 0x00, 0x00, 0x00, 0x00, 0x00 }
905 }, {
906 PCI_VENDOR_HIFN,
907 PCI_PRODUCT_HIFN_7956,
908 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
909 0x00, 0x00, 0x00, 0x00, 0x00 }
910 }, {
911 PCI_VENDOR_NETSEC,
912 PCI_PRODUCT_NETSEC_7751,
913 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
914 0x00, 0x00, 0x00, 0x00, 0x00 }
915 }, {
916 PCI_VENDOR_INVERTEX,
917 PCI_PRODUCT_INVERTEX_AEON,
918 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
919 0x00, 0x00, 0x00, 0x00, 0x00 }
920 }, {
921 PCI_VENDOR_HIFN,
922 PCI_PRODUCT_HIFN_7811,
923 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
924 0x00, 0x00, 0x00, 0x00, 0x00 }
925 }, {
926 /*
927 * Other vendors share this PCI ID as well, such as
928 * http://www.powercrypt.com, and obviously they also
929 * use the same key.
930 */
931 PCI_VENDOR_HIFN,
932 PCI_PRODUCT_HIFN_7751,
933 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
934 0x00, 0x00, 0x00, 0x00, 0x00 }
935 },
936 };
937
938 /*
939 * Checks to see if crypto is already enabled. If crypto isn't enable,
940 * "hifn_enable_crypto" is called to enable it. The check is important,
941 * as enabling crypto twice will lock the board.
942 */
943 static int
944 hifn_enable_crypto(struct hifn_softc *sc)
945 {
946 u_int32_t dmacfg, ramcfg, encl, addr, i;
947 char *offtbl = NULL;
948
949 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
950 if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
951 pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
952 offtbl = pci2id[i].card_id;
953 break;
954 }
955 }
956 if (offtbl == NULL) {
957 device_printf(sc->sc_dev, "Unknown card!\n");
958 return (1);
959 }
960
961 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
962 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
963
964 /*
965 * The RAM config register's encrypt level bit needs to be set before
966 * every read performed on the encryption level register.
967 */
968 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
969
970 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
971
972 /*
973 * Make sure we don't re-unlock. Two unlocks kills chip until the
974 * next reboot.
975 */
976 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
977 #ifdef HIFN_DEBUG
978 if (hifn_debug)
979 device_printf(sc->sc_dev,
980 "Strong crypto already enabled!\n");
981 #endif
982 goto report;
983 }
984
985 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
986 #ifdef HIFN_DEBUG
987 if (hifn_debug)
988 device_printf(sc->sc_dev,
989 "Unknown encryption level 0x%x\n", encl);
990 #endif
991 return 1;
992 }
993
994 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
995 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
996 DELAY(1000);
997 addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
998 DELAY(1000);
999 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
1000 DELAY(1000);
1001
1002 for (i = 0; i <= 12; i++) {
1003 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
1004 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
1005
1006 DELAY(1000);
1007 }
1008
1009 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1010 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1011
1012 #ifdef HIFN_DEBUG
1013 if (hifn_debug) {
1014 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
1015 device_printf(sc->sc_dev, "Engine is permanently "
1016 "locked until next system reset!\n");
1017 else
1018 device_printf(sc->sc_dev, "Engine enabled "
1019 "successfully!\n");
1020 }
1021 #endif
1022
1023 report:
1024 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
1025 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
1026
1027 switch (encl) {
1028 case HIFN_PUSTAT_ENA_1:
1029 case HIFN_PUSTAT_ENA_2:
1030 break;
1031 case HIFN_PUSTAT_ENA_0:
1032 default:
1033 device_printf(sc->sc_dev, "disabled");
1034 break;
1035 }
1036
1037 return 0;
1038 }
1039
1040 /*
1041 * Give initial values to the registers listed in the "Register Space"
1042 * section of the HIFN Software Development reference manual.
1043 */
1044 static void
1045 hifn_init_pci_registers(struct hifn_softc *sc)
1046 {
1047 /* write fixed values needed by the Initialization registers */
1048 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1049 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1050 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1051
1052 /* write all 4 ring address registers */
1053 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
1054 offsetof(struct hifn_dma, cmdr[0]));
1055 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
1056 offsetof(struct hifn_dma, srcr[0]));
1057 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
1058 offsetof(struct hifn_dma, dstr[0]));
1059 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
1060 offsetof(struct hifn_dma, resr[0]));
1061
1062 DELAY(2000);
1063
1064 /* write status register */
1065 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1066 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1067 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1068 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1069 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1070 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1071 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1072 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1073 HIFN_DMACSR_S_WAIT |
1074 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1075 HIFN_DMACSR_C_WAIT |
1076 HIFN_DMACSR_ENGINE |
1077 ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1078 HIFN_DMACSR_PUBDONE : 0) |
1079 ((sc->sc_flags & HIFN_IS_7811) ?
1080 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1081
1082 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1083 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1084 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1085 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1086 ((sc->sc_flags & HIFN_IS_7811) ?
1087 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1088 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1089 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1090
1091
1092 if (sc->sc_flags & HIFN_IS_7956) {
1093 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1094 HIFN_PUCNFG_TCALLPHASES |
1095 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1096 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956);
1097 } else {
1098 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1099 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1100 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1101 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1102 }
1103
1104 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1105 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1106 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1107 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1108 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1109 }
1110
1111 /*
1112 * The maximum number of sessions supported by the card
1113 * is dependent on the amount of context ram, which
1114 * encryption algorithms are enabled, and how compression
1115 * is configured. This should be configured before this
1116 * routine is called.
1117 */
1118 static void
1119 hifn_sessions(struct hifn_softc *sc)
1120 {
1121 u_int32_t pucnfg;
1122 int ctxsize;
1123
1124 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1125
1126 if (pucnfg & HIFN_PUCNFG_COMPSING) {
1127 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1128 ctxsize = 128;
1129 else
1130 ctxsize = 512;
1131 /*
1132 * 7955/7956 has internal context memory of 32K
1133 */
1134 if (sc->sc_flags & HIFN_IS_7956)
1135 sc->sc_maxses = 32768 / ctxsize;
1136 else
1137 sc->sc_maxses = 1 +
1138 ((sc->sc_ramsize - 32768) / ctxsize);
1139 } else
1140 sc->sc_maxses = sc->sc_ramsize / 16384;
1141
1142 if (sc->sc_maxses > 2048)
1143 sc->sc_maxses = 2048;
1144 }
1145
1146 /*
1147 * Determine ram type (sram or dram). Board should be just out of a reset
1148 * state when this is called.
1149 */
1150 static int
1151 hifn_ramtype(struct hifn_softc *sc)
1152 {
1153 u_int8_t data[8], dataexpect[8];
1154 int i;
1155
1156 for (i = 0; i < sizeof(data); i++)
1157 data[i] = dataexpect[i] = 0x55;
1158 if (hifn_writeramaddr(sc, 0, data))
1159 return (-1);
1160 if (hifn_readramaddr(sc, 0, data))
1161 return (-1);
1162 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1163 sc->sc_drammodel = 1;
1164 return (0);
1165 }
1166
1167 for (i = 0; i < sizeof(data); i++)
1168 data[i] = dataexpect[i] = 0xaa;
1169 if (hifn_writeramaddr(sc, 0, data))
1170 return (-1);
1171 if (hifn_readramaddr(sc, 0, data))
1172 return (-1);
1173 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1174 sc->sc_drammodel = 1;
1175 return (0);
1176 }
1177
1178 return (0);
1179 }
1180
1181 #define HIFN_SRAM_MAX (32 << 20)
1182 #define HIFN_SRAM_STEP_SIZE 16384
1183 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1184
1185 static int
1186 hifn_sramsize(struct hifn_softc *sc)
1187 {
1188 u_int32_t a;
1189 u_int8_t data[8];
1190 u_int8_t dataexpect[sizeof(data)];
1191 int32_t i;
1192
1193 for (i = 0; i < sizeof(data); i++)
1194 data[i] = dataexpect[i] = i ^ 0x5a;
1195
1196 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1197 a = i * HIFN_SRAM_STEP_SIZE;
1198 bcopy(&i, data, sizeof(i));
1199 hifn_writeramaddr(sc, a, data);
1200 }
1201
1202 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1203 a = i * HIFN_SRAM_STEP_SIZE;
1204 bcopy(&i, dataexpect, sizeof(i));
1205 if (hifn_readramaddr(sc, a, data) < 0)
1206 return (0);
1207 if (bcmp(data, dataexpect, sizeof(data)) != 0)
1208 return (0);
1209 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1210 }
1211
1212 return (0);
1213 }
1214
1215 /*
1216 * XXX For dram boards, one should really try all of the
1217 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG
1218 * is already set up correctly.
1219 */
1220 static int
1221 hifn_dramsize(struct hifn_softc *sc)
1222 {
1223 u_int32_t cnfg;
1224
1225 if (sc->sc_flags & HIFN_IS_7956) {
1226 /*
1227 * 7955/7956 have a fixed internal ram of only 32K.
1228 */
1229 sc->sc_ramsize = 32768;
1230 } else {
1231 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1232 HIFN_PUCNFG_DRAMMASK;
1233 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1234 }
1235 return (0);
1236 }
1237
1238 static void
1239 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
1240 {
1241 struct hifn_dma *dma = sc->sc_dma;
1242
1243 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1244 dma->cmdi = 0;
1245 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1246 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1247 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1248 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1249 }
1250 *cmdp = dma->cmdi++;
1251 dma->cmdk = dma->cmdi;
1252
1253 if (dma->srci == HIFN_D_SRC_RSIZE) {
1254 dma->srci = 0;
1255 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1256 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1257 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1258 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1259 }
1260 *srcp = dma->srci++;
1261 dma->srck = dma->srci;
1262
1263 if (dma->dsti == HIFN_D_DST_RSIZE) {
1264 dma->dsti = 0;
1265 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1266 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1267 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1268 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1269 }
1270 *dstp = dma->dsti++;
1271 dma->dstk = dma->dsti;
1272
1273 if (dma->resi == HIFN_D_RES_RSIZE) {
1274 dma->resi = 0;
1275 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1276 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1277 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1278 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1279 }
1280 *resp = dma->resi++;
1281 dma->resk = dma->resi;
1282 }
1283
1284 static int
1285 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1286 {
1287 struct hifn_dma *dma = sc->sc_dma;
1288 hifn_base_command_t wc;
1289 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1290 int r, cmdi, resi, srci, dsti;
1291
1292 wc.masks = htole16(3 << 13);
1293 wc.session_num = htole16(addr >> 14);
1294 wc.total_source_count = htole16(8);
1295 wc.total_dest_count = htole16(addr & 0x3fff);
1296
1297 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1298
1299 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1300 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1301 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1302
1303 /* build write command */
1304 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1305 *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
1306 bcopy(data, &dma->test_src, sizeof(dma->test_src));
1307
1308 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
1309 + offsetof(struct hifn_dma, test_src));
1310 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
1311 + offsetof(struct hifn_dma, test_dst));
1312
1313 dma->cmdr[cmdi].l = htole32(16 | masks);
1314 dma->srcr[srci].l = htole32(8 | masks);
1315 dma->dstr[dsti].l = htole32(4 | masks);
1316 dma->resr[resi].l = htole32(4 | masks);
1317
1318 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1319 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1320
1321 for (r = 10000; r >= 0; r--) {
1322 DELAY(10);
1323 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1324 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1325 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1326 break;
1327 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1328 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1329 }
1330 if (r == 0) {
1331 device_printf(sc->sc_dev, "writeramaddr -- "
1332 "result[%d](addr %d) still valid\n", resi, addr);
1333 r = -1;
1334 return (-1);
1335 } else
1336 r = 0;
1337
1338 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1339 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1340 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1341
1342 return (r);
1343 }
1344
1345 static int
1346 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1347 {
1348 struct hifn_dma *dma = sc->sc_dma;
1349 hifn_base_command_t rc;
1350 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1351 int r, cmdi, srci, dsti, resi;
1352
1353 rc.masks = htole16(2 << 13);
1354 rc.session_num = htole16(addr >> 14);
1355 rc.total_source_count = htole16(addr & 0x3fff);
1356 rc.total_dest_count = htole16(8);
1357
1358 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1359
1360 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1361 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1362 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1363
1364 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1365 *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
1366
1367 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
1368 offsetof(struct hifn_dma, test_src));
1369 dma->test_src = 0;
1370 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr +
1371 offsetof(struct hifn_dma, test_dst));
1372 dma->test_dst = 0;
1373 dma->cmdr[cmdi].l = htole32(8 | masks);
1374 dma->srcr[srci].l = htole32(8 | masks);
1375 dma->dstr[dsti].l = htole32(8 | masks);
1376 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1377
1378 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1379 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1380
1381 for (r = 10000; r >= 0; r--) {
1382 DELAY(10);
1383 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1384 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1385 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1386 break;
1387 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1388 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1389 }
1390 if (r == 0) {
1391 device_printf(sc->sc_dev, "readramaddr -- "
1392 "result[%d](addr %d) still valid\n", resi, addr);
1393 r = -1;
1394 } else {
1395 r = 0;
1396 bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
1397 }
1398
1399 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1400 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1401 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1402
1403 return (r);
1404 }
1405
1406 /*
1407 * Initialize the descriptor rings.
1408 */
1409 static void
1410 hifn_init_dma(struct hifn_softc *sc)
1411 {
1412 struct hifn_dma *dma = sc->sc_dma;
1413 int i;
1414
1415 hifn_set_retry(sc);
1416
1417 /* initialize static pointer values */
1418 for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1419 dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
1420 offsetof(struct hifn_dma, command_bufs[i][0]));
1421 for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1422 dma->resr[i].p = htole32(sc->sc_dma_physaddr +
1423 offsetof(struct hifn_dma, result_bufs[i][0]));
1424
1425 dma->cmdr[HIFN_D_CMD_RSIZE].p =
1426 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
1427 dma->srcr[HIFN_D_SRC_RSIZE].p =
1428 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
1429 dma->dstr[HIFN_D_DST_RSIZE].p =
1430 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
1431 dma->resr[HIFN_D_RES_RSIZE].p =
1432 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
1433
1434 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
1435 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
1436 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
1437 }
1438
1439 /*
1440 * Writes out the raw command buffer space. Returns the
1441 * command buffer size.
1442 */
1443 static u_int
1444 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1445 {
1446 u_int8_t *buf_pos;
1447 hifn_base_command_t *base_cmd;
1448 hifn_mac_command_t *mac_cmd;
1449 hifn_crypt_command_t *cry_cmd;
1450 int using_mac, using_crypt, len, ivlen;
1451 u_int32_t dlen, slen;
1452
1453 buf_pos = buf;
1454 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1455 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1456
1457 base_cmd = (hifn_base_command_t *)buf_pos;
1458 base_cmd->masks = htole16(cmd->base_masks);
1459 slen = cmd->src_mapsize;
1460 if (cmd->sloplen)
1461 dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
1462 else
1463 dlen = cmd->dst_mapsize;
1464 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1465 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1466 dlen >>= 16;
1467 slen >>= 16;
1468 base_cmd->session_num = htole16(cmd->session_num |
1469 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1470 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1471 buf_pos += sizeof(hifn_base_command_t);
1472
1473 if (using_mac) {
1474 mac_cmd = (hifn_mac_command_t *)buf_pos;
1475 dlen = cmd->maccrd->crd_len;
1476 mac_cmd->source_count = htole16(dlen & 0xffff);
1477 dlen >>= 16;
1478 mac_cmd->masks = htole16(cmd->mac_masks |
1479 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1480 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1481 mac_cmd->reserved = 0;
1482 buf_pos += sizeof(hifn_mac_command_t);
1483 }
1484
1485 if (using_crypt) {
1486 cry_cmd = (hifn_crypt_command_t *)buf_pos;
1487 dlen = cmd->enccrd->crd_len;
1488 cry_cmd->source_count = htole16(dlen & 0xffff);
1489 dlen >>= 16;
1490 cry_cmd->masks = htole16(cmd->cry_masks |
1491 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1492 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1493 cry_cmd->reserved = 0;
1494 buf_pos += sizeof(hifn_crypt_command_t);
1495 }
1496
1497 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1498 bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
1499 buf_pos += HIFN_MAC_KEY_LENGTH;
1500 }
1501
1502 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1503 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1504 case HIFN_CRYPT_CMD_ALG_3DES:
1505 bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
1506 buf_pos += HIFN_3DES_KEY_LENGTH;
1507 break;
1508 case HIFN_CRYPT_CMD_ALG_DES:
1509 bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
1510 buf_pos += HIFN_DES_KEY_LENGTH;
1511 break;
1512 case HIFN_CRYPT_CMD_ALG_RC4:
1513 len = 256;
1514 do {
1515 int clen;
1516
1517 clen = MIN(cmd->cklen, len);
1518 bcopy(cmd->ck, buf_pos, clen);
1519 len -= clen;
1520 buf_pos += clen;
1521 } while (len > 0);
1522 bzero(buf_pos, 4);
1523 buf_pos += 4;
1524 break;
1525 case HIFN_CRYPT_CMD_ALG_AES:
1526 /*
1527 * AES keys are variable 128, 192 and
1528 * 256 bits (16, 24 and 32 bytes).
1529 */
1530 bcopy(cmd->ck, buf_pos, cmd->cklen);
1531 buf_pos += cmd->cklen;
1532 break;
1533 }
1534 }
1535
1536 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1537 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1538 case HIFN_CRYPT_CMD_ALG_AES:
1539 ivlen = HIFN_AES_IV_LENGTH;
1540 break;
1541 default:
1542 ivlen = HIFN_IV_LENGTH;
1543 break;
1544 }
1545 bcopy(cmd->iv, buf_pos, ivlen);
1546 buf_pos += ivlen;
1547 }
1548
1549 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
1550 bzero(buf_pos, 8);
1551 buf_pos += 8;
1552 }
1553
1554 return (buf_pos - buf);
1555 }
1556
1557 static int
1558 hifn_dmamap_aligned(struct hifn_operand *op)
1559 {
1560 int i;
1561
1562 for (i = 0; i < op->nsegs; i++) {
1563 if (op->segs[i].ds_addr & 3)
1564 return (0);
1565 if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
1566 return (0);
1567 }
1568 return (1);
1569 }
1570
1571 static int
1572 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1573 {
1574 struct hifn_dma *dma = sc->sc_dma;
1575 struct hifn_operand *dst = &cmd->dst;
1576 u_int32_t p, l;
1577 int idx, used = 0, i;
1578
1579 idx = dma->dsti;
1580 for (i = 0; i < dst->nsegs - 1; i++) {
1581 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1582 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1583 HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
1584 HIFN_DSTR_SYNC(sc, idx,
1585 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1586 used++;
1587
1588 if (++idx == HIFN_D_DST_RSIZE) {
1589 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1590 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1591 HIFN_DSTR_SYNC(sc, idx,
1592 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1593 idx = 0;
1594 }
1595 }
1596
1597 if (cmd->sloplen == 0) {
1598 p = dst->segs[i].ds_addr;
1599 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1600 dst->segs[i].ds_len;
1601 } else {
1602 p = sc->sc_dma_physaddr +
1603 offsetof(struct hifn_dma, slop[cmd->slopidx]);
1604 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1605 sizeof(u_int32_t);
1606
1607 if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
1608 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1609 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1610 HIFN_D_MASKDONEIRQ |
1611 (dst->segs[i].ds_len - cmd->sloplen));
1612 HIFN_DSTR_SYNC(sc, idx,
1613 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1614 used++;
1615
1616 if (++idx == HIFN_D_DST_RSIZE) {
1617 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1618 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1619 HIFN_DSTR_SYNC(sc, idx,
1620 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1621 idx = 0;
1622 }
1623 }
1624 }
1625 dma->dstr[idx].p = htole32(p);
1626 dma->dstr[idx].l = htole32(l);
1627 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1628 used++;
1629
1630 if (++idx == HIFN_D_DST_RSIZE) {
1631 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1632 HIFN_D_MASKDONEIRQ);
1633 HIFN_DSTR_SYNC(sc, idx,
1634 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1635 idx = 0;
1636 }
1637
1638 dma->dsti = idx;
1639 dma->dstu += used;
1640 return (idx);
1641 }
1642
1643 static int
1644 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1645 {
1646 struct hifn_dma *dma = sc->sc_dma;
1647 struct hifn_operand *src = &cmd->src;
1648 int idx, i;
1649 u_int32_t last = 0;
1650
1651 idx = dma->srci;
1652 for (i = 0; i < src->nsegs; i++) {
1653 if (i == src->nsegs - 1)
1654 last = HIFN_D_LAST;
1655
1656 dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
1657 dma->srcr[idx].l = htole32(src->segs[i].ds_len |
1658 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1659 HIFN_SRCR_SYNC(sc, idx,
1660 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1661
1662 if (++idx == HIFN_D_SRC_RSIZE) {
1663 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1664 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1665 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1666 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1667 idx = 0;
1668 }
1669 }
1670 dma->srci = idx;
1671 dma->srcu += src->nsegs;
1672 return (idx);
1673 }
1674
1675 static void
1676 hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1677 {
1678 struct hifn_operand *op = arg;
1679
1680 KASSERT(nsegs <= MAX_SCATTER,
1681 ("hifn_op_cb: too many DMA segments (%u > %u) "
1682 "returned when mapping operand", nsegs, MAX_SCATTER));
1683 op->mapsize = mapsize;
1684 op->nsegs = nsegs;
1685 bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1686 }
1687
1688 static int
1689 hifn_crypto(
1690 struct hifn_softc *sc,
1691 struct hifn_command *cmd,
1692 struct cryptop *crp,
1693 int hint)
1694 {
1695 struct hifn_dma *dma = sc->sc_dma;
1696 u_int32_t cmdlen;
1697 int cmdi, resi, err = 0;
1698
1699 /*
1700 * need 1 cmd, and 1 res
1701 *
1702 * NB: check this first since it's easy.
1703 */
1704 HIFN_LOCK(sc);
1705 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
1706 (dma->resu + 1) > HIFN_D_RES_RSIZE) {
1707 #ifdef HIFN_DEBUG
1708 if (hifn_debug) {
1709 device_printf(sc->sc_dev,
1710 "cmd/result exhaustion, cmdu %u resu %u\n",
1711 dma->cmdu, dma->resu);
1712 }
1713 #endif
1714 hifnstats.hst_nomem_cr++;
1715 HIFN_UNLOCK(sc);
1716 return (ERESTART);
1717 }
1718
1719 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
1720 hifnstats.hst_nomem_map++;
1721 HIFN_UNLOCK(sc);
1722 return (ENOMEM);
1723 }
1724
1725 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1726 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1727 cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1728 hifnstats.hst_nomem_load++;
1729 err = ENOMEM;
1730 goto err_srcmap1;
1731 }
1732 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1733 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1734 cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1735 hifnstats.hst_nomem_load++;
1736 err = ENOMEM;
1737 goto err_srcmap1;
1738 }
1739 } else {
1740 err = EINVAL;
1741 goto err_srcmap1;
1742 }
1743
1744 if (hifn_dmamap_aligned(&cmd->src)) {
1745 cmd->sloplen = cmd->src_mapsize & 3;
1746 cmd->dst = cmd->src;
1747 } else {
1748 if (crp->crp_flags & CRYPTO_F_IOV) {
1749 err = EINVAL;
1750 goto err_srcmap;
1751 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1752 int totlen, len;
1753 struct mbuf *m, *m0, *mlast;
1754
1755 KASSERT(cmd->dst_m == cmd->src_m,
1756 ("hifn_crypto: dst_m initialized improperly"));
1757 hifnstats.hst_unaligned++;
1758 /*
1759 * Source is not aligned on a longword boundary.
1760 * Copy the data to insure alignment. If we fail
1761 * to allocate mbufs or clusters while doing this
1762 * we return ERESTART so the operation is requeued
1763 * at the crypto later, but only if there are
1764 * ops already posted to the hardware; otherwise we
1765 * have no guarantee that we'll be re-entered.
1766 */
1767 totlen = cmd->src_mapsize;
1768 if (cmd->src_m->m_flags & M_PKTHDR) {
1769 len = MHLEN;
1770 MGETHDR(m0, M_DONTWAIT, MT_DATA);
1771 if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_DONTWAIT)) {
1772 m_free(m0);
1773 m0 = NULL;
1774 }
1775 } else {
1776 len = MLEN;
1777 MGET(m0, M_DONTWAIT, MT_DATA);
1778 }
1779 if (m0 == NULL) {
1780 hifnstats.hst_nomem_mbuf++;
1781 err = dma->cmdu ? ERESTART : ENOMEM;
1782 goto err_srcmap;
1783 }
1784 if (totlen >= MINCLSIZE) {
1785 MCLGET(m0, M_DONTWAIT);
1786 if ((m0->m_flags & M_EXT) == 0) {
1787 hifnstats.hst_nomem_mcl++;
1788 err = dma->cmdu ? ERESTART : ENOMEM;
1789 m_freem(m0);
1790 goto err_srcmap;
1791 }
1792 len = MCLBYTES;
1793 }
1794 totlen -= len;
1795 m0->m_pkthdr.len = m0->m_len = len;
1796 mlast = m0;
1797
1798 while (totlen > 0) {
1799 MGET(m, M_DONTWAIT, MT_DATA);
1800 if (m == NULL) {
1801 hifnstats.hst_nomem_mbuf++;
1802 err = dma->cmdu ? ERESTART : ENOMEM;
1803 m_freem(m0);
1804 goto err_srcmap;
1805 }
1806 len = MLEN;
1807 if (totlen >= MINCLSIZE) {
1808 MCLGET(m, M_DONTWAIT);
1809 if ((m->m_flags & M_EXT) == 0) {
1810 hifnstats.hst_nomem_mcl++;
1811 err = dma->cmdu ? ERESTART : ENOMEM;
1812 mlast->m_next = m;
1813 m_freem(m0);
1814 goto err_srcmap;
1815 }
1816 len = MCLBYTES;
1817 }
1818
1819 m->m_len = len;
1820 m0->m_pkthdr.len += len;
1821 totlen -= len;
1822
1823 mlast->m_next = m;
1824 mlast = m;
1825 }
1826 cmd->dst_m = m0;
1827 }
1828 }
1829
1830 if (cmd->dst_map == NULL) {
1831 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1832 hifnstats.hst_nomem_map++;
1833 err = ENOMEM;
1834 goto err_srcmap;
1835 }
1836 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1837 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1838 cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1839 hifnstats.hst_nomem_map++;
1840 err = ENOMEM;
1841 goto err_dstmap1;
1842 }
1843 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1844 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1845 cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1846 hifnstats.hst_nomem_load++;
1847 err = ENOMEM;
1848 goto err_dstmap1;
1849 }
1850 }
1851 }
1852
1853 #ifdef HIFN_DEBUG
1854 if (hifn_debug) {
1855 device_printf(sc->sc_dev,
1856 "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1857 READ_REG_1(sc, HIFN_1_DMA_CSR),
1858 READ_REG_1(sc, HIFN_1_DMA_IER),
1859 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1860 cmd->src_nsegs, cmd->dst_nsegs);
1861 }
1862 #endif
1863
1864 if (cmd->src_map == cmd->dst_map) {
1865 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1866 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1867 } else {
1868 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1869 BUS_DMASYNC_PREWRITE);
1870 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1871 BUS_DMASYNC_PREREAD);
1872 }
1873
1874 /*
1875 * need N src, and N dst
1876 */
1877 if ((dma->srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
1878 (dma->dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
1879 #ifdef HIFN_DEBUG
1880 if (hifn_debug) {
1881 device_printf(sc->sc_dev,
1882 "src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
1883 dma->srcu, cmd->src_nsegs,
1884 dma->dstu, cmd->dst_nsegs);
1885 }
1886 #endif
1887 hifnstats.hst_nomem_sd++;
1888 err = ERESTART;
1889 goto err_dstmap;
1890 }
1891
1892 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1893 dma->cmdi = 0;
1894 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1895 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1896 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1897 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1898 }
1899 cmdi = dma->cmdi++;
1900 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
1901 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
1902
1903 /* .p for command/result already set */
1904 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
1905 HIFN_D_MASKDONEIRQ);
1906 HIFN_CMDR_SYNC(sc, cmdi,
1907 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1908 dma->cmdu++;
1909 if (sc->sc_c_busy == 0) {
1910 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1911 sc->sc_c_busy = 1;
1912 }
1913
1914 /*
1915 * We don't worry about missing an interrupt (which a "command wait"
1916 * interrupt salvages us from), unless there is more than one command
1917 * in the queue.
1918 */
1919 if (dma->cmdu > 1) {
1920 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
1921 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1922 }
1923
1924 hifnstats.hst_ipackets++;
1925 hifnstats.hst_ibytes += cmd->src_mapsize;
1926
1927 hifn_dmamap_load_src(sc, cmd);
1928 if (sc->sc_s_busy == 0) {
1929 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1930 sc->sc_s_busy = 1;
1931 }
1932
1933 /*
1934 * Unlike other descriptors, we don't mask done interrupt from
1935 * result descriptor.
1936 */
1937 #ifdef HIFN_DEBUG
1938 if (hifn_debug)
1939 printf("load res\n");
1940 #endif
1941 if (dma->resi == HIFN_D_RES_RSIZE) {
1942 dma->resi = 0;
1943 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1944 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1945 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1946 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1947 }
1948 resi = dma->resi++;
1949 KASSERT(dma->hifn_commands[resi] == NULL,
1950 ("hifn_crypto: command slot %u busy", resi));
1951 dma->hifn_commands[resi] = cmd;
1952 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
1953 if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
1954 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1955 HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1956 sc->sc_curbatch++;
1957 if (sc->sc_curbatch > hifnstats.hst_maxbatch)
1958 hifnstats.hst_maxbatch = sc->sc_curbatch;
1959 hifnstats.hst_totbatch++;
1960 } else {
1961 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1962 HIFN_D_VALID | HIFN_D_LAST);
1963 sc->sc_curbatch = 0;
1964 }
1965 HIFN_RESR_SYNC(sc, resi,
1966 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1967 dma->resu++;
1968 if (sc->sc_r_busy == 0) {
1969 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1970 sc->sc_r_busy = 1;
1971 }
1972
1973 if (cmd->sloplen)
1974 cmd->slopidx = resi;
1975
1976 hifn_dmamap_load_dst(sc, cmd);
1977
1978 if (sc->sc_d_busy == 0) {
1979 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1980 sc->sc_d_busy = 1;
1981 }
1982
1983 #ifdef HIFN_DEBUG
1984 if (hifn_debug) {
1985 device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
1986 READ_REG_1(sc, HIFN_1_DMA_CSR),
1987 READ_REG_1(sc, HIFN_1_DMA_IER));
1988 }
1989 #endif
1990
1991 sc->sc_active = 5;
1992 HIFN_UNLOCK(sc);
1993 KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
1994 return (err); /* success */
1995
1996 err_dstmap:
1997 if (cmd->src_map != cmd->dst_map)
1998 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
1999 err_dstmap1:
2000 if (cmd->src_map != cmd->dst_map)
2001 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2002 err_srcmap:
2003 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2004 if (cmd->src_m != cmd->dst_m)
2005 m_freem(cmd->dst_m);
2006 }
2007 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2008 err_srcmap1:
2009 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2010 HIFN_UNLOCK(sc);
2011 return (err);
2012 }
2013
2014 static void
2015 hifn_tick(void* vsc)
2016 {
2017 struct hifn_softc *sc = vsc;
2018
2019 HIFN_LOCK(sc);
2020 if (sc->sc_active == 0) {
2021 struct hifn_dma *dma = sc->sc_dma;
2022 u_int32_t r = 0;
2023
2024 if (dma->cmdu == 0 && sc->sc_c_busy) {
2025 sc->sc_c_busy = 0;
2026 r |= HIFN_DMACSR_C_CTRL_DIS;
2027 }
2028 if (dma->srcu == 0 && sc->sc_s_busy) {
2029 sc->sc_s_busy = 0;
2030 r |= HIFN_DMACSR_S_CTRL_DIS;
2031 }
2032 if (dma->dstu == 0 && sc->sc_d_busy) {
2033 sc->sc_d_busy = 0;
2034 r |= HIFN_DMACSR_D_CTRL_DIS;
2035 }
2036 if (dma->resu == 0 && sc->sc_r_busy) {
2037 sc->sc_r_busy = 0;
2038 r |= HIFN_DMACSR_R_CTRL_DIS;
2039 }
2040 if (r)
2041 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
2042 } else
2043 sc->sc_active--;
2044 HIFN_UNLOCK(sc);
2045 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
2046 }
2047
2048 static void
2049 hifn_intr(void *arg)
2050 {
2051 struct hifn_softc *sc = arg;
2052 struct hifn_dma *dma;
2053 u_int32_t dmacsr, restart;
2054 int i, u;
2055
2056 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
2057
2058 /* Nothing in the DMA unit interrupted */
2059 if ((dmacsr & sc->sc_dmaier) == 0)
2060 return;
2061
2062 HIFN_LOCK(sc);
2063
2064 dma = sc->sc_dma;
2065
2066 #ifdef HIFN_DEBUG
2067 if (hifn_debug) {
2068 device_printf(sc->sc_dev,
2069 "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
2070 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
2071 dma->cmdi, dma->srci, dma->dsti, dma->resi,
2072 dma->cmdk, dma->srck, dma->dstk, dma->resk,
2073 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2074 }
2075 #endif
2076
2077 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2078
2079 if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2080 (dmacsr & HIFN_DMACSR_PUBDONE))
2081 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2082 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2083
2084 restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
2085 if (restart)
2086 device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
2087
2088 if (sc->sc_flags & HIFN_IS_7811) {
2089 if (dmacsr & HIFN_DMACSR_ILLR)
2090 device_printf(sc->sc_dev, "illegal read\n");
2091 if (dmacsr & HIFN_DMACSR_ILLW)
2092 device_printf(sc->sc_dev, "illegal write\n");
2093 }
2094
2095 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2096 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2097 if (restart) {
2098 device_printf(sc->sc_dev, "abort, resetting.\n");
2099 hifnstats.hst_abort++;
2100 hifn_abort(sc);
2101 HIFN_UNLOCK(sc);
2102 return;
2103 }
2104
2105 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
2106 /*
2107 * If no slots to process and we receive a "waiting on
2108 * command" interrupt, we disable the "waiting on command"
2109 * (by clearing it).
2110 */
2111 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2112 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2113 }
2114
2115 /* clear the rings */
2116 i = dma->resk; u = dma->resu;
2117 while (u != 0) {
2118 HIFN_RESR_SYNC(sc, i,
2119 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2120 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2121 HIFN_RESR_SYNC(sc, i,
2122 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2123 break;
2124 }
2125
2126 if (i != HIFN_D_RES_RSIZE) {
2127 struct hifn_command *cmd;
2128 u_int8_t *macbuf = NULL;
2129
2130 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2131 cmd = dma->hifn_commands[i];
2132 KASSERT(cmd != NULL,
2133 ("hifn_intr: null command slot %u", i));
2134 dma->hifn_commands[i] = NULL;
2135
2136 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2137 macbuf = dma->result_bufs[i];
2138 macbuf += 12;
2139 }
2140
2141 hifn_callback(sc, cmd, macbuf);
2142 hifnstats.hst_opackets++;
2143 u--;
2144 }
2145
2146 if (++i == (HIFN_D_RES_RSIZE + 1))
2147 i = 0;
2148 }
2149 dma->resk = i; dma->resu = u;
2150
2151 i = dma->srck; u = dma->srcu;
2152 while (u != 0) {
2153 if (i == HIFN_D_SRC_RSIZE)
2154 i = 0;
2155 HIFN_SRCR_SYNC(sc, i,
2156 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2157 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2158 HIFN_SRCR_SYNC(sc, i,
2159 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2160 break;
2161 }
2162 i++, u--;
2163 }
2164 dma->srck = i; dma->srcu = u;
2165
2166 i = dma->cmdk; u = dma->cmdu;
2167 while (u != 0) {
2168 HIFN_CMDR_SYNC(sc, i,
2169 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2170 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2171 HIFN_CMDR_SYNC(sc, i,
2172 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2173 break;
2174 }
2175 if (i != HIFN_D_CMD_RSIZE) {
2176 u--;
2177 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2178 }
2179 if (++i == (HIFN_D_CMD_RSIZE + 1))
2180 i = 0;
2181 }
2182 dma->cmdk = i; dma->cmdu = u;
2183
2184 HIFN_UNLOCK(sc);
2185
2186 if (sc->sc_needwakeup) { /* XXX check high watermark */
2187 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
2188 #ifdef HIFN_DEBUG
2189 if (hifn_debug)
2190 device_printf(sc->sc_dev,
2191 "wakeup crypto (%x) u %d/%d/%d/%d\n",
2192 sc->sc_needwakeup,
2193 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2194 #endif
2195 sc->sc_needwakeup &= ~wakeup;
2196 crypto_unblock(sc->sc_cid, wakeup);
2197 }
2198 }
2199
2200 /*
2201 * Allocate a new 'session' and return an encoded session id. 'sidp'
2202 * contains our registration id, and should contain an encoded session
2203 * id on successful allocation.
2204 */
2205 static int
2206 hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
2207 {
2208 struct cryptoini *c;
2209 struct hifn_softc *sc = arg;
2210 int i, mac = 0, cry = 0;
2211
2212 KASSERT(sc != NULL, ("hifn_newsession: null softc"));
2213 if (sidp == NULL || cri == NULL || sc == NULL)
2214 return (EINVAL);
2215
2216 for (i = 0; i < sc->sc_maxses; i++)
2217 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE)
2218 break;
2219 if (i == sc->sc_maxses)
2220 return (ENOMEM);
2221
2222 for (c = cri; c != NULL; c = c->cri_next) {
2223 switch (c->cri_alg) {
2224 case CRYPTO_MD5:
2225 case CRYPTO_SHA1:
2226 case CRYPTO_MD5_HMAC:
2227 case CRYPTO_SHA1_HMAC:
2228 if (mac)
2229 return (EINVAL);
2230 mac = 1;
2231 break;
2232 case CRYPTO_DES_CBC:
2233 case CRYPTO_3DES_CBC:
2234 case CRYPTO_AES_CBC:
2235 /* XXX this may read fewer, does it matter? */
2236 read_random(sc->sc_sessions[i].hs_iv,
2237 c->cri_alg == CRYPTO_AES_CBC ?
2238 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2239 /*FALLTHROUGH*/
2240 case CRYPTO_ARC4:
2241 if (cry)
2242 return (EINVAL);
2243 cry = 1;
2244 break;
2245 default:
2246 return (EINVAL);
2247 }
2248 }
2249 if (mac == 0 && cry == 0)
2250 return (EINVAL);
2251
2252 *sidp = HIFN_SID(device_get_unit(sc->sc_dev), i);
2253 sc->sc_sessions[i].hs_state = HS_STATE_USED;
2254
2255 return (0);
2256 }
2257
2258 /*
2259 * Deallocate a session.
2260 * XXX this routine should run a zero'd mac/encrypt key into context ram.
2261 * XXX to blow away any keys already stored there.
2262 */
2263 static int
2264 hifn_freesession(void *arg, u_int64_t tid)
2265 {
2266 struct hifn_softc *sc = arg;
2267 int session;
2268 u_int32_t sid = CRYPTO_SESID2LID(tid);
2269
2270 KASSERT(sc != NULL, ("hifn_freesession: null softc"));
2271 if (sc == NULL)
2272 return (EINVAL);
2273
2274 session = HIFN_SESSION(sid);
2275 if (session >= sc->sc_maxses)
2276 return (EINVAL);
2277
2278 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
2279 return (0);
2280 }
2281
2282 static int
2283 hifn_process(void *arg, struct cryptop *crp, int hint)
2284 {
2285 struct hifn_softc *sc = arg;
2286 struct hifn_command *cmd = NULL;
2287 int session, err, ivlen;
2288 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2289
2290 if (crp == NULL || crp->crp_callback == NULL) {
2291 hifnstats.hst_invalid++;
2292 return (EINVAL);
2293 }
2294 session = HIFN_SESSION(crp->crp_sid);
2295
2296 if (sc == NULL || session >= sc->sc_maxses) {
2297 err = EINVAL;
2298 goto errout;
2299 }
2300
2301 cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
2302 if (cmd == NULL) {
2303 hifnstats.hst_nomem++;
2304 err = ENOMEM;
2305 goto errout;
2306 }
2307
2308 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2309 cmd->src_m = (struct mbuf *)crp->crp_buf;
2310 cmd->dst_m = (struct mbuf *)crp->crp_buf;
2311 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2312 cmd->src_io = (struct uio *)crp->crp_buf;
2313 cmd->dst_io = (struct uio *)crp->crp_buf;
2314 } else {
2315 err = EINVAL;
2316 goto errout; /* XXX we don't handle contiguous buffers! */
2317 }
2318
2319 crd1 = crp->crp_desc;
2320 if (crd1 == NULL) {
2321 err = EINVAL;
2322 goto errout;
2323 }
2324 crd2 = crd1->crd_next;
2325
2326 if (crd2 == NULL) {
2327 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
2328 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2329 crd1->crd_alg == CRYPTO_SHA1 ||
2330 crd1->crd_alg == CRYPTO_MD5) {
2331 maccrd = crd1;
2332 enccrd = NULL;
2333 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2334 crd1->crd_alg == CRYPTO_3DES_CBC ||
2335 crd1->crd_alg == CRYPTO_AES_CBC ||
2336 crd1->crd_alg == CRYPTO_ARC4) {
2337 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2338 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2339 maccrd = NULL;
2340 enccrd = crd1;
2341 } else {
2342 err = EINVAL;
2343 goto errout;
2344 }
2345 } else {
2346 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
2347 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2348 crd1->crd_alg == CRYPTO_MD5 ||
2349 crd1->crd_alg == CRYPTO_SHA1) &&
2350 (crd2->crd_alg == CRYPTO_DES_CBC ||
2351 crd2->crd_alg == CRYPTO_3DES_CBC ||
2352 crd2->crd_alg == CRYPTO_AES_CBC ||
2353 crd2->crd_alg == CRYPTO_ARC4) &&
2354 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2355 cmd->base_masks = HIFN_BASE_CMD_DECODE;
2356 maccrd = crd1;
2357 enccrd = crd2;
2358 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2359 crd1->crd_alg == CRYPTO_ARC4 ||
2360 crd1->crd_alg == CRYPTO_3DES_CBC ||
2361 crd1->crd_alg == CRYPTO_AES_CBC) &&
2362 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
2363 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
2364 crd2->crd_alg == CRYPTO_MD5 ||
2365 crd2->crd_alg == CRYPTO_SHA1) &&
2366 (crd1->crd_flags & CRD_F_ENCRYPT)) {
2367 enccrd = crd1;
2368 maccrd = crd2;
2369 } else {
2370 /*
2371 * We cannot order the 7751 as requested
2372 */
2373 err = EINVAL;
2374 goto errout;
2375 }
2376 }
2377
2378 if (enccrd) {
2379 cmd->enccrd = enccrd;
2380 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2381 switch (enccrd->crd_alg) {
2382 case CRYPTO_ARC4:
2383 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2384 if ((enccrd->crd_flags & CRD_F_ENCRYPT)
2385 != sc->sc_sessions[session].hs_prev_op)
2386 sc->sc_sessions[session].hs_state =
2387 HS_STATE_USED;
2388 break;
2389 case CRYPTO_DES_CBC:
2390 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2391 HIFN_CRYPT_CMD_MODE_CBC |
2392 HIFN_CRYPT_CMD_NEW_IV;
2393 break;
2394 case CRYPTO_3DES_CBC:
2395 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2396 HIFN_CRYPT_CMD_MODE_CBC |
2397 HIFN_CRYPT_CMD_NEW_IV;
2398 break;
2399 case CRYPTO_AES_CBC:
2400 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2401 HIFN_CRYPT_CMD_MODE_CBC |
2402 HIFN_CRYPT_CMD_NEW_IV;
2403 break;
2404 default:
2405 err = EINVAL;
2406 goto errout;
2407 }
2408 if (enccrd->crd_alg != CRYPTO_ARC4) {
2409 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2410 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2411 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2412 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2413 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2414 else
2415 bcopy(sc->sc_sessions[session].hs_iv,
2416 cmd->iv, ivlen);
2417
2418 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2419 == 0) {
2420 if (crp->crp_flags & CRYPTO_F_IMBUF)
2421 m_copyback(cmd->src_m,
2422 enccrd->crd_inject,
2423 ivlen, cmd->iv);
2424 else if (crp->crp_flags & CRYPTO_F_IOV)
2425 cuio_copyback(cmd->src_io,
2426 enccrd->crd_inject,
2427 ivlen, cmd->iv);
2428 }
2429 } else {
2430 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2431 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2432 else if (crp->crp_flags & CRYPTO_F_IMBUF)
2433 m_copydata(cmd->src_m,
2434 enccrd->crd_inject, ivlen, cmd->iv);
2435 else if (crp->crp_flags & CRYPTO_F_IOV)
2436 cuio_copydata(cmd->src_io,
2437 enccrd->crd_inject, ivlen, cmd->iv);
2438 }
2439 }
2440
2441 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
2442 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2443 cmd->ck = enccrd->crd_key;
2444 cmd->cklen = enccrd->crd_klen >> 3;
2445
2446 /*
2447 * Need to specify the size for the AES key in the masks.
2448 */
2449 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2450 HIFN_CRYPT_CMD_ALG_AES) {
2451 switch (cmd->cklen) {
2452 case 16:
2453 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2454 break;
2455 case 24:
2456 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2457 break;
2458 case 32:
2459 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2460 break;
2461 default:
2462 err = EINVAL;
2463 goto errout;
2464 }
2465 }
2466
2467 if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2468 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2469 }
2470
2471 if (maccrd) {
2472 cmd->maccrd = maccrd;
2473 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2474
2475 switch (maccrd->crd_alg) {
2476 case CRYPTO_MD5:
2477 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2478 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2479 HIFN_MAC_CMD_POS_IPSEC;
2480 break;
2481 case CRYPTO_MD5_HMAC:
2482 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2483 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2484 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2485 break;
2486 case CRYPTO_SHA1:
2487 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2488 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2489 HIFN_MAC_CMD_POS_IPSEC;
2490 break;
2491 case CRYPTO_SHA1_HMAC:
2492 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2493 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2494 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2495 break;
2496 }
2497
2498 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
2499 maccrd->crd_alg == CRYPTO_MD5_HMAC) &&
2500 sc->sc_sessions[session].hs_state == HS_STATE_USED) {
2501 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2502 bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
2503 bzero(cmd->mac + (maccrd->crd_klen >> 3),
2504 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2505 }
2506 }
2507
2508 cmd->crp = crp;
2509 cmd->session_num = session;
2510 cmd->softc = sc;
2511
2512 err = hifn_crypto(sc, cmd, crp, hint);
2513 if (!err) {
2514 if (enccrd)
2515 sc->sc_sessions[session].hs_prev_op =
2516 enccrd->crd_flags & CRD_F_ENCRYPT;
2517 if (sc->sc_sessions[session].hs_state == HS_STATE_USED)
2518 sc->sc_sessions[session].hs_state = HS_STATE_KEY;
2519 return 0;
2520 } else if (err == ERESTART) {
2521 /*
2522 * There weren't enough resources to dispatch the request
2523 * to the part. Notify the caller so they'll requeue this
2524 * request and resubmit it again soon.
2525 */
2526 #ifdef HIFN_DEBUG
2527 if (hifn_debug)
2528 device_printf(sc->sc_dev, "requeue request\n");
2529 #endif
2530 free(cmd, M_DEVBUF);
2531 sc->sc_needwakeup |= CRYPTO_SYMQ;
2532 return (err);
2533 }
2534
2535 errout:
2536 if (cmd != NULL)
2537 free(cmd, M_DEVBUF);
2538 if (err == EINVAL)
2539 hifnstats.hst_invalid++;
2540 else
2541 hifnstats.hst_nomem++;
2542 crp->crp_etype = err;
2543 crypto_done(crp);
2544 return (err);
2545 }
2546
2547 static void
2548 hifn_abort(struct hifn_softc *sc)
2549 {
2550 struct hifn_dma *dma = sc->sc_dma;
2551 struct hifn_command *cmd;
2552 struct cryptop *crp;
2553 int i, u;
2554
2555 i = dma->resk; u = dma->resu;
2556 while (u != 0) {
2557 cmd = dma->hifn_commands[i];
2558 KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
2559 dma->hifn_commands[i] = NULL;
2560 crp = cmd->crp;
2561
2562 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2563 /* Salvage what we can. */
2564 u_int8_t *macbuf;
2565
2566 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2567 macbuf = dma->result_bufs[i];
2568 macbuf += 12;
2569 } else
2570 macbuf = NULL;
2571 hifnstats.hst_opackets++;
2572 hifn_callback(sc, cmd, macbuf);
2573 } else {
2574 if (cmd->src_map == cmd->dst_map) {
2575 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2576 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2577 } else {
2578 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2579 BUS_DMASYNC_POSTWRITE);
2580 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2581 BUS_DMASYNC_POSTREAD);
2582 }
2583
2584 if (cmd->src_m != cmd->dst_m) {
2585 m_freem(cmd->src_m);
2586 crp->crp_buf = (caddr_t)cmd->dst_m;
2587 }
2588
2589 /* non-shared buffers cannot be restarted */
2590 if (cmd->src_map != cmd->dst_map) {
2591 /*
2592 * XXX should be EAGAIN, delayed until
2593 * after the reset.
2594 */
2595 crp->crp_etype = ENOMEM;
2596 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2597 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2598 } else
2599 crp->crp_etype = ENOMEM;
2600
2601 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2602 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2603
2604 free(cmd, M_DEVBUF);
2605 if (crp->crp_etype != EAGAIN)
2606 crypto_done(crp);
2607 }
2608
2609 if (++i == HIFN_D_RES_RSIZE)
2610 i = 0;
2611 u--;
2612 }
2613 dma->resk = i; dma->resu = u;
2614
2615 /* Force upload of key next time */
2616 for (i = 0; i < sc->sc_maxses; i++)
2617 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY)
2618 sc->sc_sessions[i].hs_state = HS_STATE_USED;
2619
2620 hifn_reset_board(sc, 1);
2621 hifn_init_dma(sc);
2622 hifn_init_pci_registers(sc);
2623 }
2624
2625 static void
2626 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
2627 {
2628 struct hifn_dma *dma = sc->sc_dma;
2629 struct cryptop *crp = cmd->crp;
2630 struct cryptodesc *crd;
2631 struct mbuf *m;
2632 int totlen, i, u, ivlen;
2633
2634 if (cmd->src_map == cmd->dst_map) {
2635 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2636 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2637 } else {
2638 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2639 BUS_DMASYNC_POSTWRITE);
2640 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2641 BUS_DMASYNC_POSTREAD);
2642 }
2643
2644 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2645 if (cmd->src_m != cmd->dst_m) {
2646 crp->crp_buf = (caddr_t)cmd->dst_m;
2647 totlen = cmd->src_mapsize;
2648 for (m = cmd->dst_m; m != NULL; m = m->m_next) {
2649 if (totlen < m->m_len) {
2650 m->m_len = totlen;
2651 totlen = 0;
2652 } else
2653 totlen -= m->m_len;
2654 }
2655 cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
2656 m_freem(cmd->src_m);
2657 }
2658 }
2659
2660 if (cmd->sloplen != 0) {
2661 if (crp->crp_flags & CRYPTO_F_IMBUF)
2662 m_copyback((struct mbuf *)crp->crp_buf,
2663 cmd->src_mapsize - cmd->sloplen,
2664 cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]);
2665 else if (crp->crp_flags & CRYPTO_F_IOV)
2666 cuio_copyback((struct uio *)crp->crp_buf,
2667 cmd->src_mapsize - cmd->sloplen,
2668 cmd->sloplen, (caddr_t)&dma->slop[cmd->slopidx]);
2669 }
2670
2671 i = dma->dstk; u = dma->dstu;
2672 while (u != 0) {
2673 if (i == HIFN_D_DST_RSIZE)
2674 i = 0;
2675 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2676 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2677 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2678 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2679 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2680 break;
2681 }
2682 i++, u--;
2683 }
2684 dma->dstk = i; dma->dstu = u;
2685
2686 hifnstats.hst_obytes += cmd->dst_mapsize;
2687
2688 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2689 HIFN_BASE_CMD_CRYPT) {
2690 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2691 if (crd->crd_alg != CRYPTO_DES_CBC &&
2692 crd->crd_alg != CRYPTO_3DES_CBC &&
2693 crd->crd_alg != CRYPTO_AES_CBC)
2694 continue;
2695 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
2696 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2697 if (crp->crp_flags & CRYPTO_F_IMBUF)
2698 m_copydata((struct mbuf *)crp->crp_buf,
2699 crd->crd_skip + crd->crd_len - ivlen, ivlen,
2700 cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2701 else if (crp->crp_flags & CRYPTO_F_IOV) {
2702 cuio_copydata((struct uio *)crp->crp_buf,
2703 crd->crd_skip + crd->crd_len - ivlen, ivlen,
2704 cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2705 }
2706 break;
2707 }
2708 }
2709
2710 if (macbuf != NULL) {
2711 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2712 int len;
2713
2714 if (crd->crd_alg == CRYPTO_MD5)
2715 len = 16;
2716 else if (crd->crd_alg == CRYPTO_SHA1)
2717 len = 20;
2718 else if (crd->crd_alg == CRYPTO_MD5_HMAC ||
2719 crd->crd_alg == CRYPTO_SHA1_HMAC)
2720 len = 12;
2721 else
2722 continue;
2723
2724 if (crp->crp_flags & CRYPTO_F_IMBUF)
2725 m_copyback((struct mbuf *)crp->crp_buf,
2726 crd->crd_inject, len, macbuf);
2727 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac)
2728 bcopy((caddr_t)macbuf, crp->crp_mac, len);
2729 break;
2730 }
2731 }
2732
2733 if (cmd->src_map != cmd->dst_map) {
2734 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2735 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2736 }
2737 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2738 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2739 free(cmd, M_DEVBUF);
2740 crypto_done(crp);
2741 }
2742
2743 /*
2744 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
2745 * and Group 1 registers; avoid conditions that could create
2746 * burst writes by doing a read in between the writes.
2747 *
2748 * NB: The read we interpose is always to the same register;
2749 * we do this because reading from an arbitrary (e.g. last)
2750 * register may not always work.
2751 */
2752 static void
2753 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2754 {
2755 if (sc->sc_flags & HIFN_IS_7811) {
2756 if (sc->sc_bar0_lastreg == reg - 4)
2757 bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
2758 sc->sc_bar0_lastreg = reg;
2759 }
2760 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
2761 }
2762
2763 static void
2764 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2765 {
2766 if (sc->sc_flags & HIFN_IS_7811) {
2767 if (sc->sc_bar1_lastreg == reg - 4)
2768 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
2769 sc->sc_bar1_lastreg = reg;
2770 }
2771 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
2772 }
Cache object: 996ece839459d07d9cb6c74200f95d13
|