FreeBSD/Linux Kernel Cross Reference
sys/dev/sec/sec.c
1 /*-
2 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
17 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
19 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
21 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
22 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 /*
27 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
28 * 3.0 are supported.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: releng/8.4/sys/dev/sec/sec.c 230714 2012-01-29 01:22:48Z marius $");
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/bus.h>
37 #include <sys/endian.h>
38 #include <sys/kernel.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/mutex.h>
44 #include <sys/random.h>
45 #include <sys/rman.h>
46
47 #include <machine/bus.h>
48 #include <machine/ocpbus.h>
49 #include <machine/resource.h>
50
51 #include <opencrypto/cryptodev.h>
52 #include "cryptodev_if.h"
53
54 #include <dev/sec/sec.h>
55
56 static int sec_probe(device_t dev);
57 static int sec_attach(device_t dev);
58 static int sec_detach(device_t dev);
59 static int sec_suspend(device_t dev);
60 static int sec_resume(device_t dev);
61 static int sec_shutdown(device_t dev);
62 static void sec_primary_intr(void *arg);
63 static void sec_secondary_intr(void *arg);
64 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires,
65 void **ihand, int *irid, driver_intr_t handler, const char *iname);
66 static void sec_release_intr(struct sec_softc *sc, struct resource *ires,
67 void *ihand, int irid, const char *iname);
68 static int sec_controller_reset(struct sec_softc *sc);
69 static int sec_channel_reset(struct sec_softc *sc, int channel, int full);
70 static int sec_init(struct sec_softc *sc);
71 static int sec_alloc_dma_mem(struct sec_softc *sc,
72 struct sec_dma_mem *dma_mem, bus_size_t size);
73 static int sec_desc_map_dma(struct sec_softc *sc,
74 struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
75 struct sec_desc_map_info *sdmi);
76 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
77 static void sec_enqueue(struct sec_softc *sc);
78 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
79 int channel);
80 static int sec_eu_channel(struct sec_softc *sc, int eu);
81 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
82 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
83 static int sec_make_pointer_direct(struct sec_softc *sc,
84 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
85 static int sec_alloc_session(struct sec_softc *sc);
86 static int sec_newsession(device_t dev, u_int32_t *sidp,
87 struct cryptoini *cri);
88 static int sec_freesession(device_t dev, uint64_t tid);
89 static int sec_process(device_t dev, struct cryptop *crp, int hint);
90 static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
91 struct cryptoini **mac);
92 static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
93 struct cryptodesc **mac);
94 static int sec_build_common_ns_desc(struct sec_softc *sc,
95 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
96 struct cryptodesc *enc, int buftype);
97 static int sec_build_common_s_desc(struct sec_softc *sc,
98 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
99 struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
100
101 static struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid);
102 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
103
104 /* AESU */
105 static int sec_aesu_newsession(struct sec_softc *sc,
106 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
107 static int sec_aesu_make_desc(struct sec_softc *sc,
108 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
109 int buftype);
110
111 /* DEU */
112 static int sec_deu_newsession(struct sec_softc *sc,
113 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
114 static int sec_deu_make_desc(struct sec_softc *sc,
115 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
116 int buftype);
117
118 /* MDEU */
119 static int sec_mdeu_can_handle(u_int alg);
120 static int sec_mdeu_config(struct cryptodesc *crd,
121 u_int *eu, u_int *mode, u_int *hashlen);
122 static int sec_mdeu_newsession(struct sec_softc *sc,
123 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
124 static int sec_mdeu_make_desc(struct sec_softc *sc,
125 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
126 int buftype);
127
128 static device_method_t sec_methods[] = {
129 /* Device interface */
130 DEVMETHOD(device_probe, sec_probe),
131 DEVMETHOD(device_attach, sec_attach),
132 DEVMETHOD(device_detach, sec_detach),
133
134 DEVMETHOD(device_suspend, sec_suspend),
135 DEVMETHOD(device_resume, sec_resume),
136 DEVMETHOD(device_shutdown, sec_shutdown),
137
138 /* Crypto methods */
139 DEVMETHOD(cryptodev_newsession, sec_newsession),
140 DEVMETHOD(cryptodev_freesession,sec_freesession),
141 DEVMETHOD(cryptodev_process, sec_process),
142
143 DEVMETHOD_END
144 };
145 static driver_t sec_driver = {
146 "sec",
147 sec_methods,
148 sizeof(struct sec_softc),
149 };
150
151 static devclass_t sec_devclass;
152 DRIVER_MODULE(sec, ocpbus, sec_driver, sec_devclass, 0, 0);
153 MODULE_DEPEND(sec, crypto, 1, 1, 1);
154
155 static struct sec_eu_methods sec_eus[] = {
156 {
157 sec_aesu_newsession,
158 sec_aesu_make_desc,
159 },
160 {
161 sec_deu_newsession,
162 sec_deu_make_desc,
163 },
164 {
165 sec_mdeu_newsession,
166 sec_mdeu_make_desc,
167 },
168 { NULL, NULL }
169 };
170
171 static inline void
172 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
173 {
174
175 /* Sync only if dma memory is valid */
176 if (dma_mem->dma_vaddr != NULL)
177 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
178 }
179
180 static inline void
181 sec_free_session(struct sec_softc *sc, struct sec_session *ses)
182 {
183
184 SEC_LOCK(sc, sessions);
185 ses->ss_used = 0;
186 SEC_UNLOCK(sc, sessions);
187 }
188
189 static inline void *
190 sec_get_pointer_data(struct sec_desc *desc, u_int n)
191 {
192
193 return (desc->sd_ptr_dmem[n].dma_vaddr);
194 }
195
196 static int
197 sec_probe(device_t dev)
198 {
199 struct sec_softc *sc;
200 device_t parent;
201 uintptr_t devtype;
202 uint64_t id;
203 int error;
204
205 parent = device_get_parent(dev);
206 error = BUS_READ_IVAR(parent, dev, OCPBUS_IVAR_DEVTYPE, &devtype);
207 if (error)
208 return (error);
209
210 if (devtype != OCPBUS_DEVTYPE_SEC)
211 return (ENXIO);
212
213 sc = device_get_softc(dev);
214
215 sc->sc_rrid = 0;
216 sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid,
217 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE);
218
219 if (sc->sc_rres == NULL)
220 return (ENXIO);
221
222 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
223 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
224
225 id = SEC_READ(sc, SEC_ID);
226
227 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
228
229 switch (id) {
230 case SEC_20_ID:
231 device_set_desc(dev, "Freescale Security Engine 2.0");
232 sc->sc_version = 2;
233 break;
234 case SEC_30_ID:
235 device_set_desc(dev, "Freescale Security Engine 3.0");
236 sc->sc_version = 3;
237 break;
238 default:
239 device_printf(dev, "unknown SEC ID 0x%016llx!\n", id);
240 return (ENXIO);
241 }
242
243 return (0);
244 }
245
246 static int
247 sec_attach(device_t dev)
248 {
249 struct sec_softc *sc;
250 struct sec_hw_lt *lt;
251 int error = 0;
252 int i;
253
254 sc = device_get_softc(dev);
255 sc->sc_dev = dev;
256 sc->sc_blocked = 0;
257 sc->sc_shutdown = 0;
258
259 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
260 if (sc->sc_cid < 0) {
261 device_printf(dev, "could not get crypto driver ID!\n");
262 return (ENXIO);
263 }
264
265 /* Init locks */
266 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
267 "SEC Controller lock", MTX_DEF);
268 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
269 "SEC Descriptors lock", MTX_DEF);
270 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
271 "SEC Sessions lock", MTX_DEF);
272
273 /* Allocate I/O memory for SEC registers */
274 sc->sc_rrid = 0;
275 sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid,
276 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE);
277
278 if (sc->sc_rres == NULL) {
279 device_printf(dev, "could not allocate I/O memory!\n");
280 goto fail1;
281 }
282
283 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
284 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
285
286 /* Setup interrupts */
287 sc->sc_pri_irid = 0;
288 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
289 &sc->sc_pri_irid, sec_primary_intr, "primary");
290
291 if (error)
292 goto fail2;
293
294 sc->sc_sec_irid = 1;
295 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
296 &sc->sc_sec_irid, sec_secondary_intr, "secondary");
297
298 if (error)
299 goto fail3;
300
301 /* Alloc DMA memory for descriptors and link tables */
302 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
303 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
304
305 if (error)
306 goto fail4;
307
308 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
309 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
310
311 if (error)
312 goto fail5;
313
314 /* Fill in descriptors and link tables */
315 for (i = 0; i < SEC_DESCRIPTORS; i++) {
316 sc->sc_desc[i].sd_desc =
317 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
318 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
319 (i * sizeof(struct sec_hw_desc));
320 }
321
322 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
323 sc->sc_lt[i].sl_lt =
324 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
325 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
326 (i * sizeof(struct sec_hw_lt));
327 }
328
329 /* Last entry in link table is used to create a circle */
330 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
331 lt->shl_length = 0;
332 lt->shl_r = 0;
333 lt->shl_n = 1;
334 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
335
336 /* Init descriptor and link table queues pointers */
337 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
338 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
339 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
340 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
341 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
342 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
343 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
344 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
345
346 /* Create masks for fast checks */
347 sc->sc_int_error_mask = 0;
348 for (i = 0; i < SEC_CHANNELS; i++)
349 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
350
351 switch (sc->sc_version) {
352 case 2:
353 sc->sc_channel_idle_mask =
354 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
355 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
356 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
357 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
358 break;
359 case 3:
360 sc->sc_channel_idle_mask =
361 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
362 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
363 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
364 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
365 break;
366 }
367
368 /* Init hardware */
369 error = sec_init(sc);
370
371 if (error)
372 goto fail6;
373
374 /* Register in OCF (AESU) */
375 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
376
377 /* Register in OCF (DEU) */
378 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
379 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
380
381 /* Register in OCF (MDEU) */
382 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
383 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
384 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
385 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
386 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
387 if (sc->sc_version >= 3) {
388 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
389 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
390 }
391
392 return (0);
393
394 fail6:
395 sec_free_dma_mem(&(sc->sc_lt_dmem));
396 fail5:
397 sec_free_dma_mem(&(sc->sc_desc_dmem));
398 fail4:
399 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
400 sc->sc_sec_irid, "secondary");
401 fail3:
402 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
403 sc->sc_pri_irid, "primary");
404 fail2:
405 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
406 fail1:
407 mtx_destroy(&sc->sc_controller_lock);
408 mtx_destroy(&sc->sc_descriptors_lock);
409 mtx_destroy(&sc->sc_sessions_lock);
410
411 return (ENXIO);
412 }
413
414 static int
415 sec_detach(device_t dev)
416 {
417 struct sec_softc *sc = device_get_softc(dev);
418 int i, error, timeout = SEC_TIMEOUT;
419
420 /* Prepare driver to shutdown */
421 SEC_LOCK(sc, descriptors);
422 sc->sc_shutdown = 1;
423 SEC_UNLOCK(sc, descriptors);
424
425 /* Wait until all queued processing finishes */
426 while (1) {
427 SEC_LOCK(sc, descriptors);
428 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
429 SEC_UNLOCK(sc, descriptors);
430
431 if (i == 0)
432 break;
433
434 if (timeout < 0) {
435 device_printf(dev, "queue flush timeout!\n");
436
437 /* DMA can be still active - stop it */
438 for (i = 0; i < SEC_CHANNELS; i++)
439 sec_channel_reset(sc, i, 1);
440
441 break;
442 }
443
444 timeout -= 1000;
445 DELAY(1000);
446 }
447
448 /* Disable interrupts */
449 SEC_WRITE(sc, SEC_IER, 0);
450
451 /* Unregister from OCF */
452 crypto_unregister_all(sc->sc_cid);
453
454 /* Free DMA memory */
455 for (i = 0; i < SEC_DESCRIPTORS; i++)
456 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
457
458 sec_free_dma_mem(&(sc->sc_lt_dmem));
459 sec_free_dma_mem(&(sc->sc_desc_dmem));
460
461 /* Release interrupts */
462 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
463 sc->sc_pri_irid, "primary");
464 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
465 sc->sc_sec_irid, "secondary");
466
467 /* Release memory */
468 if (sc->sc_rres) {
469 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
470 sc->sc_rres);
471 if (error)
472 device_printf(dev, "bus_release_resource() failed for"
473 " I/O memory, error %d\n", error);
474
475 sc->sc_rres = NULL;
476 }
477
478 mtx_destroy(&sc->sc_controller_lock);
479 mtx_destroy(&sc->sc_descriptors_lock);
480 mtx_destroy(&sc->sc_sessions_lock);
481
482 return (0);
483 }
484
485 static int
486 sec_suspend(device_t dev)
487 {
488
489 return (0);
490 }
491
492 static int
493 sec_resume(device_t dev)
494 {
495
496 return (0);
497 }
498
499 static int
500 sec_shutdown(device_t dev)
501 {
502
503 return (0);
504 }
505
506 static int
507 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
508 int *irid, driver_intr_t handler, const char *iname)
509 {
510 int error;
511
512 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
513 RF_ACTIVE);
514
515 if ((*ires) == NULL) {
516 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
517 return (ENXIO);
518 }
519
520 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
521 NULL, handler, sc, ihand);
522
523 if (error) {
524 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
525 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
526 device_printf(sc->sc_dev, "could not release %s IRQ\n",
527 iname);
528
529 (*ires) = NULL;
530 return (error);
531 }
532
533 return (0);
534 }
535
536 static void
537 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
538 int irid, const char *iname)
539 {
540 int error;
541
542 if (ires == NULL)
543 return;
544
545 error = bus_teardown_intr(sc->sc_dev, ires, ihand);
546 if (error)
547 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
548 " IRQ, error %d\n", iname, error);
549
550 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
551 if (error)
552 device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
553 " IRQ, error %d\n", iname, error);
554 }
555
556 static void
557 sec_primary_intr(void *arg)
558 {
559 struct sec_softc *sc = arg;
560 struct sec_desc *desc;
561 uint64_t isr;
562 int i, wakeup = 0;
563
564 SEC_LOCK(sc, controller);
565
566 /* Check for errors */
567 isr = SEC_READ(sc, SEC_ISR);
568 if (isr & sc->sc_int_error_mask) {
569 /* Check each channel for error */
570 for (i = 0; i < SEC_CHANNELS; i++) {
571 if ((isr & SEC_INT_CH_ERR(i)) == 0)
572 continue;
573
574 device_printf(sc->sc_dev,
575 "I/O error on channel %i!\n", i);
576
577 /* Find and mark problematic descriptor */
578 desc = sec_find_desc(sc, SEC_READ(sc,
579 SEC_CHAN_CDPR(i)));
580
581 if (desc != NULL)
582 desc->sd_error = EIO;
583
584 /* Do partial channel reset */
585 sec_channel_reset(sc, i, 0);
586 }
587 }
588
589 /* ACK interrupt */
590 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
591
592 SEC_UNLOCK(sc, controller);
593 SEC_LOCK(sc, descriptors);
594
595 /* Handle processed descriptors */
596 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
597
598 while (SEC_QUEUED_DESC_CNT(sc) > 0) {
599 desc = SEC_GET_QUEUED_DESC(sc);
600
601 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
602 SEC_PUT_BACK_QUEUED_DESC(sc);
603 break;
604 }
605
606 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
607 BUS_DMASYNC_PREWRITE);
608
609 desc->sd_crp->crp_etype = desc->sd_error;
610 crypto_done(desc->sd_crp);
611
612 SEC_DESC_FREE_POINTERS(desc);
613 SEC_DESC_FREE_LT(sc, desc);
614 SEC_DESC_QUEUED2FREE(sc);
615 }
616
617 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
618
619 if (!sc->sc_shutdown) {
620 wakeup = sc->sc_blocked;
621 sc->sc_blocked = 0;
622 }
623
624 SEC_UNLOCK(sc, descriptors);
625
626 /* Enqueue ready descriptors in hardware */
627 sec_enqueue(sc);
628
629 if (wakeup)
630 crypto_unblock(sc->sc_cid, wakeup);
631 }
632
633 static void
634 sec_secondary_intr(void *arg)
635 {
636 struct sec_softc *sc = arg;
637
638 device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
639 sec_primary_intr(arg);
640 }
641
642 static int
643 sec_controller_reset(struct sec_softc *sc)
644 {
645 int timeout = SEC_TIMEOUT;
646
647 /* Reset Controller */
648 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
649
650 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
651 DELAY(1000);
652 timeout -= 1000;
653
654 if (timeout < 0) {
655 device_printf(sc->sc_dev, "timeout while waiting for "
656 "device reset!\n");
657 return (ETIMEDOUT);
658 }
659 }
660
661 return (0);
662 }
663
664 static int
665 sec_channel_reset(struct sec_softc *sc, int channel, int full)
666 {
667 int timeout = SEC_TIMEOUT;
668 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
669 uint64_t reg;
670
671 /* Reset Channel */
672 reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
673 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
674
675 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
676 DELAY(1000);
677 timeout -= 1000;
678
679 if (timeout < 0) {
680 device_printf(sc->sc_dev, "timeout while waiting for "
681 "channel reset!\n");
682 return (ETIMEDOUT);
683 }
684 }
685
686 if (full) {
687 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
688
689 switch(sc->sc_version) {
690 case 2:
691 reg |= SEC_CHAN_CCR_CDWE;
692 break;
693 case 3:
694 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
695 break;
696 }
697
698 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
699 }
700
701 return (0);
702 }
703
704 static int
705 sec_init(struct sec_softc *sc)
706 {
707 uint64_t reg;
708 int error, i;
709
710 /* Reset controller twice to clear all pending interrupts */
711 error = sec_controller_reset(sc);
712 if (error)
713 return (error);
714
715 error = sec_controller_reset(sc);
716 if (error)
717 return (error);
718
719 /* Reset channels */
720 for (i = 0; i < SEC_CHANNELS; i++) {
721 error = sec_channel_reset(sc, i, 1);
722 if (error)
723 return (error);
724 }
725
726 /* Enable Interrupts */
727 reg = SEC_INT_ITO;
728 for (i = 0; i < SEC_CHANNELS; i++)
729 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
730
731 SEC_WRITE(sc, SEC_IER, reg);
732
733 return (error);
734 }
735
736 static void
737 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
738 {
739 struct sec_dma_mem *dma_mem = arg;
740
741 if (error)
742 return;
743
744 KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
745 dma_mem->dma_paddr = segs->ds_addr;
746 }
747
748 static void
749 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
750 int error)
751 {
752 struct sec_desc_map_info *sdmi = arg;
753 struct sec_softc *sc = sdmi->sdmi_sc;
754 struct sec_lt *lt = NULL;
755 bus_addr_t addr;
756 bus_size_t size;
757 int i;
758
759 SEC_LOCK_ASSERT(sc, descriptors);
760
761 if (error)
762 return;
763
764 for (i = 0; i < nseg; i++) {
765 addr = segs[i].ds_addr;
766 size = segs[i].ds_len;
767
768 /* Skip requested offset */
769 if (sdmi->sdmi_offset >= size) {
770 sdmi->sdmi_offset -= size;
771 continue;
772 }
773
774 addr += sdmi->sdmi_offset;
775 size -= sdmi->sdmi_offset;
776 sdmi->sdmi_offset = 0;
777
778 /* Do not link more than requested */
779 if (sdmi->sdmi_size < size)
780 size = sdmi->sdmi_size;
781
782 lt = SEC_ALLOC_LT_ENTRY(sc);
783 lt->sl_lt->shl_length = size;
784 lt->sl_lt->shl_r = 0;
785 lt->sl_lt->shl_n = 0;
786 lt->sl_lt->shl_ptr = addr;
787
788 if (sdmi->sdmi_lt_first == NULL)
789 sdmi->sdmi_lt_first = lt;
790
791 sdmi->sdmi_lt_used += 1;
792
793 if ((sdmi->sdmi_size -= size) == 0)
794 break;
795 }
796
797 sdmi->sdmi_lt_last = lt;
798 }
799
800 static void
801 sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
802 bus_size_t size, int error)
803 {
804
805 sec_dma_map_desc_cb(arg, segs, nseg, error);
806 }
807
808 static int
809 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
810 bus_size_t size)
811 {
812 int error;
813
814 if (dma_mem->dma_vaddr != NULL)
815 return (EBUSY);
816
817 error = bus_dma_tag_create(NULL, /* parent */
818 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
819 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
820 BUS_SPACE_MAXADDR, /* highaddr */
821 NULL, NULL, /* filtfunc, filtfuncarg */
822 size, 1, /* maxsize, nsegments */
823 size, 0, /* maxsegsz, flags */
824 NULL, NULL, /* lockfunc, lockfuncarg */
825 &(dma_mem->dma_tag)); /* dmat */
826
827 if (error) {
828 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
829 " %i!\n", error);
830 goto err1;
831 }
832
833 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
834 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
835
836 if (error) {
837 device_printf(sc->sc_dev, "failed to allocate DMA safe"
838 " memory, error %i!\n", error);
839 goto err2;
840 }
841
842 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
843 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
844 BUS_DMA_NOWAIT);
845
846 if (error) {
847 device_printf(sc->sc_dev, "cannot get address of the DMA"
848 " memory, error %i\n", error);
849 goto err3;
850 }
851
852 dma_mem->dma_is_map = 0;
853 return (0);
854
855 err3:
856 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
857 err2:
858 bus_dma_tag_destroy(dma_mem->dma_tag);
859 err1:
860 dma_mem->dma_vaddr = NULL;
861 return(error);
862 }
863
864 static int
865 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
866 bus_size_t size, int type, struct sec_desc_map_info *sdmi)
867 {
868 int error;
869
870 if (dma_mem->dma_vaddr != NULL)
871 return (EBUSY);
872
873 switch (type) {
874 case SEC_MEMORY:
875 break;
876 case SEC_UIO:
877 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
878 break;
879 case SEC_MBUF:
880 size = m_length((struct mbuf*)mem, NULL);
881 break;
882 default:
883 return (EINVAL);
884 }
885
886 error = bus_dma_tag_create(NULL, /* parent */
887 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
888 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
889 BUS_SPACE_MAXADDR, /* highaddr */
890 NULL, NULL, /* filtfunc, filtfuncarg */
891 size, /* maxsize */
892 SEC_FREE_LT_CNT(sc), /* nsegments */
893 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */
894 NULL, NULL, /* lockfunc, lockfuncarg */
895 &(dma_mem->dma_tag)); /* dmat */
896
897 if (error) {
898 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
899 " %i!\n", error);
900 dma_mem->dma_vaddr = NULL;
901 return (error);
902 }
903
904 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
905
906 if (error) {
907 device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
908 "\n", error);
909 bus_dma_tag_destroy(dma_mem->dma_tag);
910 return (error);
911 }
912
913 switch (type) {
914 case SEC_MEMORY:
915 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
916 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
917 break;
918 case SEC_UIO:
919 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
920 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
921 break;
922 case SEC_MBUF:
923 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
924 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
925 break;
926 }
927
928 if (error) {
929 device_printf(sc->sc_dev, "cannot get address of the DMA"
930 " memory, error %i!\n", error);
931 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
932 bus_dma_tag_destroy(dma_mem->dma_tag);
933 return (error);
934 }
935
936 dma_mem->dma_is_map = 1;
937 dma_mem->dma_vaddr = mem;
938
939 return (0);
940 }
941
942 static void
943 sec_free_dma_mem(struct sec_dma_mem *dma_mem)
944 {
945
946 /* Check for double free */
947 if (dma_mem->dma_vaddr == NULL)
948 return;
949
950 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
951
952 if (dma_mem->dma_is_map)
953 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
954 else
955 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
956 dma_mem->dma_map);
957
958 bus_dma_tag_destroy(dma_mem->dma_tag);
959 dma_mem->dma_vaddr = NULL;
960 }
961
962 static int
963 sec_eu_channel(struct sec_softc *sc, int eu)
964 {
965 uint64_t reg;
966 int channel = 0;
967
968 SEC_LOCK_ASSERT(sc, controller);
969
970 reg = SEC_READ(sc, SEC_EUASR);
971
972 switch (eu) {
973 case SEC_EU_AFEU:
974 channel = SEC_EUASR_AFEU(reg);
975 break;
976 case SEC_EU_DEU:
977 channel = SEC_EUASR_DEU(reg);
978 break;
979 case SEC_EU_MDEU_A:
980 case SEC_EU_MDEU_B:
981 channel = SEC_EUASR_MDEU(reg);
982 break;
983 case SEC_EU_RNGU:
984 channel = SEC_EUASR_RNGU(reg);
985 break;
986 case SEC_EU_PKEU:
987 channel = SEC_EUASR_PKEU(reg);
988 break;
989 case SEC_EU_AESU:
990 channel = SEC_EUASR_AESU(reg);
991 break;
992 case SEC_EU_KEU:
993 channel = SEC_EUASR_KEU(reg);
994 break;
995 case SEC_EU_CRCU:
996 channel = SEC_EUASR_CRCU(reg);
997 break;
998 }
999
1000 return (channel - 1);
1001 }
1002
1003 static int
1004 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
1005 {
1006 u_int fflvl = SEC_MAX_FIFO_LEVEL;
1007 uint64_t reg;
1008 int i;
1009
1010 SEC_LOCK_ASSERT(sc, controller);
1011
1012 /* Find free channel if have not got one */
1013 if (channel < 0) {
1014 for (i = 0; i < SEC_CHANNELS; i++) {
1015 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
1016
1017 if ((reg & sc->sc_channel_idle_mask) == 0) {
1018 channel = i;
1019 break;
1020 }
1021 }
1022 }
1023
1024 /* There is no free channel */
1025 if (channel < 0)
1026 return (-1);
1027
1028 /* Check FIFO level on selected channel */
1029 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
1030
1031 switch(sc->sc_version) {
1032 case 2:
1033 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
1034 break;
1035 case 3:
1036 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
1037 break;
1038 }
1039
1040 if (fflvl >= SEC_MAX_FIFO_LEVEL)
1041 return (-1);
1042
1043 /* Enqueue descriptor in channel */
1044 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
1045
1046 return (channel);
1047 }
1048
1049 static void
1050 sec_enqueue(struct sec_softc *sc)
1051 {
1052 struct sec_desc *desc;
1053 int ch0, ch1;
1054
1055 SEC_LOCK(sc, descriptors);
1056 SEC_LOCK(sc, controller);
1057
1058 while (SEC_READY_DESC_CNT(sc) > 0) {
1059 desc = SEC_GET_READY_DESC(sc);
1060
1061 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
1062 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
1063
1064 /*
1065 * Both EU are used by the same channel.
1066 * Enqueue descriptor in channel used by busy EUs.
1067 */
1068 if (ch0 >= 0 && ch0 == ch1) {
1069 if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
1070 SEC_DESC_READY2QUEUED(sc);
1071 continue;
1072 }
1073 }
1074
1075 /*
1076 * Only one EU is free.
1077 * Enqueue descriptor in channel used by busy EU.
1078 */
1079 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
1080 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
1081 >= 0) {
1082 SEC_DESC_READY2QUEUED(sc);
1083 continue;
1084 }
1085 }
1086
1087 /*
1088 * Both EU are free.
1089 * Enqueue descriptor in first free channel.
1090 */
1091 if (ch0 < 0 && ch1 < 0) {
1092 if (sec_enqueue_desc(sc, desc, -1) >= 0) {
1093 SEC_DESC_READY2QUEUED(sc);
1094 continue;
1095 }
1096 }
1097
1098 /* Current descriptor can not be queued at the moment */
1099 SEC_PUT_BACK_READY_DESC(sc);
1100 break;
1101 }
1102
1103 SEC_UNLOCK(sc, controller);
1104 SEC_UNLOCK(sc, descriptors);
1105 }
1106
1107 static struct sec_desc *
1108 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
1109 {
1110 struct sec_desc *desc = NULL;
1111 int i;
1112
1113 SEC_LOCK_ASSERT(sc, descriptors);
1114
1115 for (i = 0; i < SEC_CHANNELS; i++) {
1116 if (sc->sc_desc[i].sd_desc_paddr == paddr) {
1117 desc = &(sc->sc_desc[i]);
1118 break;
1119 }
1120 }
1121
1122 return (desc);
1123 }
1124
1125 static int
1126 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
1127 bus_addr_t data, bus_size_t dsize)
1128 {
1129 struct sec_hw_desc_ptr *ptr;
1130
1131 SEC_LOCK_ASSERT(sc, descriptors);
1132
1133 ptr = &(desc->sd_desc->shd_pointer[n]);
1134 ptr->shdp_length = dsize;
1135 ptr->shdp_extent = 0;
1136 ptr->shdp_j = 0;
1137 ptr->shdp_ptr = data;
1138
1139 return (0);
1140 }
1141
1142 static int
1143 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
1144 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
1145 {
1146 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
1147 struct sec_hw_desc_ptr *ptr;
1148 int error;
1149
1150 SEC_LOCK_ASSERT(sc, descriptors);
1151
1152 /* For flat memory map only requested region */
1153 if (dtype == SEC_MEMORY) {
1154 data = (uint8_t*)(data) + doffset;
1155 sdmi.sdmi_offset = 0;
1156 }
1157
1158 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
1159 dtype, &sdmi);
1160
1161 if (error)
1162 return (error);
1163
1164 sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
1165 desc->sd_lt_used += sdmi.sdmi_lt_used;
1166
1167 ptr = &(desc->sd_desc->shd_pointer[n]);
1168 ptr->shdp_length = dsize;
1169 ptr->shdp_extent = 0;
1170 ptr->shdp_j = 1;
1171 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
1172
1173 return (0);
1174 }
1175
1176 static int
1177 sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
1178 struct cryptoini **mac)
1179 {
1180 struct cryptoini *e, *m;
1181
1182 e = cri;
1183 m = cri->cri_next;
1184
1185 /* We can haldle only two operations */
1186 if (m && m->cri_next)
1187 return (EINVAL);
1188
1189 if (sec_mdeu_can_handle(e->cri_alg)) {
1190 cri = m;
1191 m = e;
1192 e = cri;
1193 }
1194
1195 if (m && !sec_mdeu_can_handle(m->cri_alg))
1196 return (EINVAL);
1197
1198 *enc = e;
1199 *mac = m;
1200
1201 return (0);
1202 }
1203
1204 static int
1205 sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
1206 struct cryptodesc **mac)
1207 {
1208 struct cryptodesc *e, *m, *t;
1209
1210 e = crp->crp_desc;
1211 m = e->crd_next;
1212
1213 /* We can haldle only two operations */
1214 if (m && m->crd_next)
1215 return (EINVAL);
1216
1217 if (sec_mdeu_can_handle(e->crd_alg)) {
1218 t = m;
1219 m = e;
1220 e = t;
1221 }
1222
1223 if (m && !sec_mdeu_can_handle(m->crd_alg))
1224 return (EINVAL);
1225
1226 *enc = e;
1227 *mac = m;
1228
1229 return (0);
1230 }
1231
1232 static int
1233 sec_alloc_session(struct sec_softc *sc)
1234 {
1235 struct sec_session *ses = NULL;
1236 int sid = -1;
1237 u_int i;
1238
1239 SEC_LOCK(sc, sessions);
1240
1241 for (i = 0; i < SEC_MAX_SESSIONS; i++) {
1242 if (sc->sc_sessions[i].ss_used == 0) {
1243 ses = &(sc->sc_sessions[i]);
1244 ses->ss_used = 1;
1245 ses->ss_ivlen = 0;
1246 ses->ss_klen = 0;
1247 ses->ss_mklen = 0;
1248 sid = i;
1249 break;
1250 }
1251 }
1252
1253 SEC_UNLOCK(sc, sessions);
1254
1255 return (sid);
1256 }
1257
1258 static struct sec_session *
1259 sec_get_session(struct sec_softc *sc, u_int sid)
1260 {
1261 struct sec_session *ses;
1262
1263 if (sid >= SEC_MAX_SESSIONS)
1264 return (NULL);
1265
1266 SEC_LOCK(sc, sessions);
1267
1268 ses = &(sc->sc_sessions[sid]);
1269
1270 if (ses->ss_used == 0)
1271 ses = NULL;
1272
1273 SEC_UNLOCK(sc, sessions);
1274
1275 return (ses);
1276 }
1277
1278 static int
1279 sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
1280 {
1281 struct sec_softc *sc = device_get_softc(dev);
1282 struct sec_eu_methods *eu = sec_eus;
1283 struct cryptoini *enc = NULL;
1284 struct cryptoini *mac = NULL;
1285 struct sec_session *ses;
1286 int error = -1;
1287 int sid;
1288
1289 error = sec_split_cri(cri, &enc, &mac);
1290 if (error)
1291 return (error);
1292
1293 /* Check key lengths */
1294 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
1295 return (E2BIG);
1296
1297 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
1298 return (E2BIG);
1299
1300 /* Only SEC 3.0 supports digests larger than 256 bits */
1301 if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
1302 return (E2BIG);
1303
1304 sid = sec_alloc_session(sc);
1305 if (sid < 0)
1306 return (ENOMEM);
1307
1308 ses = sec_get_session(sc, sid);
1309
1310 /* Find EU for this session */
1311 while (eu->sem_make_desc != NULL) {
1312 error = eu->sem_newsession(sc, ses, enc, mac);
1313 if (error >= 0)
1314 break;
1315
1316 eu++;
1317 }
1318
1319 /* If not found, return EINVAL */
1320 if (error < 0) {
1321 sec_free_session(sc, ses);
1322 return (EINVAL);
1323 }
1324
1325 /* Save cipher key */
1326 if (enc && enc->cri_key) {
1327 ses->ss_klen = enc->cri_klen / 8;
1328 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
1329 }
1330
1331 /* Save digest key */
1332 if (mac && mac->cri_key) {
1333 ses->ss_mklen = mac->cri_klen / 8;
1334 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
1335 }
1336
1337 ses->ss_eu = eu;
1338 *sidp = sid;
1339
1340 return (0);
1341 }
1342
1343 static int
1344 sec_freesession(device_t dev, uint64_t tid)
1345 {
1346 struct sec_softc *sc = device_get_softc(dev);
1347 struct sec_session *ses;
1348 int error = 0;
1349
1350 ses = sec_get_session(sc, CRYPTO_SESID2LID(tid));
1351 if (ses == NULL)
1352 return (EINVAL);
1353
1354 sec_free_session(sc, ses);
1355
1356 return (error);
1357 }
1358
1359 static int
1360 sec_process(device_t dev, struct cryptop *crp, int hint)
1361 {
1362 struct sec_softc *sc = device_get_softc(dev);
1363 struct sec_desc *desc = NULL;
1364 struct cryptodesc *mac, *enc;
1365 struct sec_session *ses;
1366 int buftype, error = 0;
1367
1368 /* Check Session ID */
1369 ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid));
1370 if (ses == NULL) {
1371 crp->crp_etype = EINVAL;
1372 crypto_done(crp);
1373 return (0);
1374 }
1375
1376 /* Check for input length */
1377 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
1378 crp->crp_etype = E2BIG;
1379 crypto_done(crp);
1380 return (0);
1381 }
1382
1383 /* Get descriptors */
1384 if (sec_split_crp(crp, &enc, &mac)) {
1385 crp->crp_etype = EINVAL;
1386 crypto_done(crp);
1387 return (0);
1388 }
1389
1390 SEC_LOCK(sc, descriptors);
1391 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1392
1393 /* Block driver if there is no free descriptors or we are going down */
1394 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
1395 sc->sc_blocked |= CRYPTO_SYMQ;
1396 SEC_UNLOCK(sc, descriptors);
1397 return (ERESTART);
1398 }
1399
1400 /* Prepare descriptor */
1401 desc = SEC_GET_FREE_DESC(sc);
1402 desc->sd_lt_used = 0;
1403 desc->sd_error = 0;
1404 desc->sd_crp = crp;
1405
1406 if (crp->crp_flags & CRYPTO_F_IOV)
1407 buftype = SEC_UIO;
1408 else if (crp->crp_flags & CRYPTO_F_IMBUF)
1409 buftype = SEC_MBUF;
1410 else
1411 buftype = SEC_MEMORY;
1412
1413 if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
1414 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1415 memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
1416 ses->ss_ivlen);
1417 else
1418 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
1419
1420 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
1421 crypto_copyback(crp->crp_flags, crp->crp_buf,
1422 enc->crd_inject, ses->ss_ivlen,
1423 desc->sd_desc->shd_iv);
1424 } else if (enc) {
1425 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1426 memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
1427 ses->ss_ivlen);
1428 else
1429 crypto_copydata(crp->crp_flags, crp->crp_buf,
1430 enc->crd_inject, ses->ss_ivlen,
1431 desc->sd_desc->shd_iv);
1432 }
1433
1434 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
1435 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
1436 ses->ss_klen = enc->crd_klen / 8;
1437 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
1438 } else
1439 error = E2BIG;
1440 }
1441
1442 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
1443 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
1444 ses->ss_mklen = mac->crd_klen / 8;
1445 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
1446 } else
1447 error = E2BIG;
1448 }
1449
1450 if (!error) {
1451 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
1452 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
1453
1454 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
1455 }
1456
1457 if (error) {
1458 SEC_DESC_FREE_POINTERS(desc);
1459 SEC_DESC_PUT_BACK_LT(sc, desc);
1460 SEC_PUT_BACK_FREE_DESC(sc);
1461 SEC_UNLOCK(sc, descriptors);
1462 crp->crp_etype = error;
1463 crypto_done(crp);
1464 return (0);
1465 }
1466
1467 /*
1468 * Skip DONE interrupt if this is not last request in burst, but only
1469 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
1470 * signaling on each descriptor.
1471 */
1472 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
1473 desc->sd_desc->shd_dn = 0;
1474 else
1475 desc->sd_desc->shd_dn = 1;
1476
1477 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1478 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
1479 BUS_DMASYNC_POSTWRITE);
1480 SEC_DESC_FREE2READY(sc);
1481 SEC_UNLOCK(sc, descriptors);
1482
1483 /* Enqueue ready descriptors in hardware */
1484 sec_enqueue(sc);
1485
1486 return (0);
1487 }
1488
1489 static int
1490 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
1491 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
1492 int buftype)
1493 {
1494 struct sec_hw_desc *hd = desc->sd_desc;
1495 int error;
1496
1497 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1498 hd->shd_eu_sel1 = SEC_EU_NONE;
1499 hd->shd_mode1 = 0;
1500
1501 /* Pointer 0: NULL */
1502 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1503 if (error)
1504 return (error);
1505
1506 /* Pointer 1: IV IN */
1507 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
1508 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
1509 if (error)
1510 return (error);
1511
1512 /* Pointer 2: Cipher Key */
1513 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1514 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
1515 if (error)
1516 return (error);
1517
1518 /* Pointer 3: Data IN */
1519 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
1520 enc->crd_len, buftype);
1521 if (error)
1522 return (error);
1523
1524 /* Pointer 4: Data OUT */
1525 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
1526 enc->crd_len, buftype);
1527 if (error)
1528 return (error);
1529
1530 /* Pointer 5: IV OUT (Not used: NULL) */
1531 error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
1532 if (error)
1533 return (error);
1534
1535 /* Pointer 6: NULL */
1536 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1537
1538 return (error);
1539 }
1540
1541 static int
1542 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
1543 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
1544 struct cryptodesc *mac, int buftype)
1545 {
1546 struct sec_hw_desc *hd = desc->sd_desc;
1547 u_int eu, mode, hashlen;
1548 int error;
1549
1550 if (mac->crd_len < enc->crd_len)
1551 return (EINVAL);
1552
1553 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
1554 return (EINVAL);
1555
1556 error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
1557 if (error)
1558 return (error);
1559
1560 hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
1561 hd->shd_eu_sel1 = eu;
1562 hd->shd_mode1 = mode;
1563
1564 /* Pointer 0: HMAC Key */
1565 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
1566 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
1567 if (error)
1568 return (error);
1569
1570 /* Pointer 1: HMAC-Only Data IN */
1571 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
1572 mac->crd_len - enc->crd_len, buftype);
1573 if (error)
1574 return (error);
1575
1576 /* Pointer 2: Cipher Key */
1577 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1578 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
1579 if (error)
1580 return (error);
1581
1582 /* Pointer 3: IV IN */
1583 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
1584 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
1585 if (error)
1586 return (error);
1587
1588 /* Pointer 4: Data IN */
1589 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
1590 enc->crd_len, buftype);
1591 if (error)
1592 return (error);
1593
1594 /* Pointer 5: Data OUT */
1595 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
1596 enc->crd_len, buftype);
1597 if (error)
1598 return (error);
1599
1600 /* Pointer 6: HMAC OUT */
1601 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
1602 hashlen, buftype);
1603
1604 return (error);
1605 }
1606
1607 /* AESU */
1608
1609 static int
1610 sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
1611 struct cryptoini *enc, struct cryptoini *mac)
1612 {
1613
1614 if (enc == NULL)
1615 return (-1);
1616
1617 if (enc->cri_alg != CRYPTO_AES_CBC)
1618 return (-1);
1619
1620 ses->ss_ivlen = AES_BLOCK_LEN;
1621
1622 return (0);
1623 }
1624
1625 static int
1626 sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1627 struct sec_desc *desc, struct cryptop *crp, int buftype)
1628 {
1629 struct sec_hw_desc *hd = desc->sd_desc;
1630 struct cryptodesc *enc, *mac;
1631 int error;
1632
1633 error = sec_split_crp(crp, &enc, &mac);
1634 if (error)
1635 return (error);
1636
1637 if (!enc)
1638 return (EINVAL);
1639
1640 hd->shd_eu_sel0 = SEC_EU_AESU;
1641 hd->shd_mode0 = SEC_AESU_MODE_CBC;
1642
1643 if (enc->crd_alg != CRYPTO_AES_CBC)
1644 return (EINVAL);
1645
1646 if (enc->crd_flags & CRD_F_ENCRYPT) {
1647 hd->shd_mode0 |= SEC_AESU_MODE_ED;
1648 hd->shd_dir = 0;
1649 } else
1650 hd->shd_dir = 1;
1651
1652 if (mac)
1653 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
1654 buftype);
1655 else
1656 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
1657 buftype);
1658
1659 return (error);
1660 }
1661
1662 /* DEU */
1663
1664 static int
1665 sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
1666 struct cryptoini *enc, struct cryptoini *mac)
1667 {
1668
1669 if (enc == NULL)
1670 return (-1);
1671
1672 switch (enc->cri_alg) {
1673 case CRYPTO_DES_CBC:
1674 case CRYPTO_3DES_CBC:
1675 break;
1676 default:
1677 return (-1);
1678 }
1679
1680 ses->ss_ivlen = DES_BLOCK_LEN;
1681
1682 return (0);
1683 }
1684
1685 static int
1686 sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1687 struct sec_desc *desc, struct cryptop *crp, int buftype)
1688 {
1689 struct sec_hw_desc *hd = desc->sd_desc;
1690 struct cryptodesc *enc, *mac;
1691 int error;
1692
1693 error = sec_split_crp(crp, &enc, &mac);
1694 if (error)
1695 return (error);
1696
1697 if (!enc)
1698 return (EINVAL);
1699
1700 hd->shd_eu_sel0 = SEC_EU_DEU;
1701 hd->shd_mode0 = SEC_DEU_MODE_CBC;
1702
1703 switch (enc->crd_alg) {
1704 case CRYPTO_3DES_CBC:
1705 hd->shd_mode0 |= SEC_DEU_MODE_TS;
1706 break;
1707 case CRYPTO_DES_CBC:
1708 break;
1709 default:
1710 return (EINVAL);
1711 }
1712
1713 if (enc->crd_flags & CRD_F_ENCRYPT) {
1714 hd->shd_mode0 |= SEC_DEU_MODE_ED;
1715 hd->shd_dir = 0;
1716 } else
1717 hd->shd_dir = 1;
1718
1719 if (mac)
1720 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
1721 buftype);
1722 else
1723 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
1724 buftype);
1725
1726 return (error);
1727 }
1728
1729 /* MDEU */
1730
1731 static int
1732 sec_mdeu_can_handle(u_int alg)
1733 {
1734 switch (alg) {
1735 case CRYPTO_MD5:
1736 case CRYPTO_SHA1:
1737 case CRYPTO_MD5_HMAC:
1738 case CRYPTO_SHA1_HMAC:
1739 case CRYPTO_SHA2_256_HMAC:
1740 case CRYPTO_SHA2_384_HMAC:
1741 case CRYPTO_SHA2_512_HMAC:
1742 return (1);
1743 default:
1744 return (0);
1745 }
1746 }
1747
1748 static int
1749 sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
1750 {
1751
1752 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
1753 *eu = SEC_EU_NONE;
1754
1755 switch (crd->crd_alg) {
1756 case CRYPTO_MD5_HMAC:
1757 *mode |= SEC_MDEU_MODE_HMAC;
1758 /* FALLTHROUGH */
1759 case CRYPTO_MD5:
1760 *eu = SEC_EU_MDEU_A;
1761 *mode |= SEC_MDEU_MODE_MD5;
1762 *hashlen = MD5_HASH_LEN;
1763 break;
1764 case CRYPTO_SHA1_HMAC:
1765 *mode |= SEC_MDEU_MODE_HMAC;
1766 /* FALLTHROUGH */
1767 case CRYPTO_SHA1:
1768 *eu = SEC_EU_MDEU_A;
1769 *mode |= SEC_MDEU_MODE_SHA1;
1770 *hashlen = SHA1_HASH_LEN;
1771 break;
1772 case CRYPTO_SHA2_256_HMAC:
1773 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
1774 *eu = SEC_EU_MDEU_A;
1775 break;
1776 case CRYPTO_SHA2_384_HMAC:
1777 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
1778 *eu = SEC_EU_MDEU_B;
1779 break;
1780 case CRYPTO_SHA2_512_HMAC:
1781 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
1782 *eu = SEC_EU_MDEU_B;
1783 break;
1784 default:
1785 return (EINVAL);
1786 }
1787
1788 if (*mode & SEC_MDEU_MODE_HMAC)
1789 *hashlen = SEC_HMAC_HASH_LEN;
1790
1791 return (0);
1792 }
1793
1794 static int
1795 sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
1796 struct cryptoini *enc, struct cryptoini *mac)
1797 {
1798
1799 if (mac && sec_mdeu_can_handle(mac->cri_alg))
1800 return (0);
1801
1802 return (-1);
1803 }
1804
1805 static int
1806 sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1807 struct sec_desc *desc, struct cryptop *crp, int buftype)
1808 {
1809 struct cryptodesc *enc, *mac;
1810 struct sec_hw_desc *hd = desc->sd_desc;
1811 u_int eu, mode, hashlen;
1812 int error;
1813
1814 error = sec_split_crp(crp, &enc, &mac);
1815 if (error)
1816 return (error);
1817
1818 if (enc)
1819 return (EINVAL);
1820
1821 error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
1822 if (error)
1823 return (error);
1824
1825 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1826 hd->shd_eu_sel0 = eu;
1827 hd->shd_mode0 = mode;
1828 hd->shd_eu_sel1 = SEC_EU_NONE;
1829 hd->shd_mode1 = 0;
1830
1831 /* Pointer 0: NULL */
1832 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1833 if (error)
1834 return (error);
1835
1836 /* Pointer 1: Context In (Not used: NULL) */
1837 error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
1838 if (error)
1839 return (error);
1840
1841 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */
1842 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
1843 error = sec_make_pointer_direct(sc, desc, 2,
1844 desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
1845 shd_mkey), ses->ss_mklen);
1846 else
1847 error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
1848
1849 if (error)
1850 return (error);
1851
1852 /* Pointer 3: Input Data */
1853 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
1854 mac->crd_len, buftype);
1855 if (error)
1856 return (error);
1857
1858 /* Pointer 4: NULL */
1859 error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
1860 if (error)
1861 return (error);
1862
1863 /* Pointer 5: Hash out */
1864 error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
1865 mac->crd_inject, hashlen, buftype);
1866 if (error)
1867 return (error);
1868
1869 /* Pointer 6: NULL */
1870 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1871
1872 return (0);
1873 }
Cache object: f3a4354d5e606e35199c48630bfd67ec
|