FreeBSD/Linux Kernel Cross Reference
sys/dev/sec/sec.c
1 /*-
2 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
17 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
19 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
21 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
22 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 /*
27 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
28 * 3.0 are supported.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: releng/10.0/sys/dev/sec/sec.c 235938 2012-05-24 21:24:23Z raj $");
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/bus.h>
37 #include <sys/endian.h>
38 #include <sys/kernel.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/mutex.h>
44 #include <sys/random.h>
45 #include <sys/rman.h>
46
47 #include <machine/bus.h>
48 #include <machine/resource.h>
49
50 #include <opencrypto/cryptodev.h>
51 #include "cryptodev_if.h"
52
53 #include <dev/ofw/ofw_bus_subr.h>
54 #include <dev/sec/sec.h>
55
56 static int sec_probe(device_t dev);
57 static int sec_attach(device_t dev);
58 static int sec_detach(device_t dev);
59 static int sec_suspend(device_t dev);
60 static int sec_resume(device_t dev);
61 static int sec_shutdown(device_t dev);
62 static void sec_primary_intr(void *arg);
63 static void sec_secondary_intr(void *arg);
64 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires,
65 void **ihand, int *irid, driver_intr_t handler, const char *iname);
66 static void sec_release_intr(struct sec_softc *sc, struct resource *ires,
67 void *ihand, int irid, const char *iname);
68 static int sec_controller_reset(struct sec_softc *sc);
69 static int sec_channel_reset(struct sec_softc *sc, int channel, int full);
70 static int sec_init(struct sec_softc *sc);
71 static int sec_alloc_dma_mem(struct sec_softc *sc,
72 struct sec_dma_mem *dma_mem, bus_size_t size);
73 static int sec_desc_map_dma(struct sec_softc *sc,
74 struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
75 struct sec_desc_map_info *sdmi);
76 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
77 static void sec_enqueue(struct sec_softc *sc);
78 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
79 int channel);
80 static int sec_eu_channel(struct sec_softc *sc, int eu);
81 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
82 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
83 static int sec_make_pointer_direct(struct sec_softc *sc,
84 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
85 static int sec_alloc_session(struct sec_softc *sc);
86 static int sec_newsession(device_t dev, u_int32_t *sidp,
87 struct cryptoini *cri);
88 static int sec_freesession(device_t dev, uint64_t tid);
89 static int sec_process(device_t dev, struct cryptop *crp, int hint);
90 static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
91 struct cryptoini **mac);
92 static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
93 struct cryptodesc **mac);
94 static int sec_build_common_ns_desc(struct sec_softc *sc,
95 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
96 struct cryptodesc *enc, int buftype);
97 static int sec_build_common_s_desc(struct sec_softc *sc,
98 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
99 struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
100
101 static struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid);
102 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
103
104 /* AESU */
105 static int sec_aesu_newsession(struct sec_softc *sc,
106 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
107 static int sec_aesu_make_desc(struct sec_softc *sc,
108 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
109 int buftype);
110
111 /* DEU */
112 static int sec_deu_newsession(struct sec_softc *sc,
113 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
114 static int sec_deu_make_desc(struct sec_softc *sc,
115 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
116 int buftype);
117
118 /* MDEU */
119 static int sec_mdeu_can_handle(u_int alg);
120 static int sec_mdeu_config(struct cryptodesc *crd,
121 u_int *eu, u_int *mode, u_int *hashlen);
122 static int sec_mdeu_newsession(struct sec_softc *sc,
123 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
124 static int sec_mdeu_make_desc(struct sec_softc *sc,
125 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
126 int buftype);
127
128 static device_method_t sec_methods[] = {
129 /* Device interface */
130 DEVMETHOD(device_probe, sec_probe),
131 DEVMETHOD(device_attach, sec_attach),
132 DEVMETHOD(device_detach, sec_detach),
133
134 DEVMETHOD(device_suspend, sec_suspend),
135 DEVMETHOD(device_resume, sec_resume),
136 DEVMETHOD(device_shutdown, sec_shutdown),
137
138 /* Crypto methods */
139 DEVMETHOD(cryptodev_newsession, sec_newsession),
140 DEVMETHOD(cryptodev_freesession,sec_freesession),
141 DEVMETHOD(cryptodev_process, sec_process),
142
143 DEVMETHOD_END
144 };
145 static driver_t sec_driver = {
146 "sec",
147 sec_methods,
148 sizeof(struct sec_softc),
149 };
150
151 static devclass_t sec_devclass;
152 DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0);
153 MODULE_DEPEND(sec, crypto, 1, 1, 1);
154
155 static struct sec_eu_methods sec_eus[] = {
156 {
157 sec_aesu_newsession,
158 sec_aesu_make_desc,
159 },
160 {
161 sec_deu_newsession,
162 sec_deu_make_desc,
163 },
164 {
165 sec_mdeu_newsession,
166 sec_mdeu_make_desc,
167 },
168 { NULL, NULL }
169 };
170
171 static inline void
172 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
173 {
174
175 /* Sync only if dma memory is valid */
176 if (dma_mem->dma_vaddr != NULL)
177 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
178 }
179
180 static inline void
181 sec_free_session(struct sec_softc *sc, struct sec_session *ses)
182 {
183
184 SEC_LOCK(sc, sessions);
185 ses->ss_used = 0;
186 SEC_UNLOCK(sc, sessions);
187 }
188
189 static inline void *
190 sec_get_pointer_data(struct sec_desc *desc, u_int n)
191 {
192
193 return (desc->sd_ptr_dmem[n].dma_vaddr);
194 }
195
196 static int
197 sec_probe(device_t dev)
198 {
199 struct sec_softc *sc;
200 uint64_t id;
201
202 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0"))
203 return (ENXIO);
204
205 sc = device_get_softc(dev);
206
207 sc->sc_rrid = 0;
208 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
209 RF_ACTIVE);
210
211 if (sc->sc_rres == NULL)
212 return (ENXIO);
213
214 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
215 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
216
217 id = SEC_READ(sc, SEC_ID);
218
219 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
220
221 switch (id) {
222 case SEC_20_ID:
223 device_set_desc(dev, "Freescale Security Engine 2.0");
224 sc->sc_version = 2;
225 break;
226 case SEC_30_ID:
227 device_set_desc(dev, "Freescale Security Engine 3.0");
228 sc->sc_version = 3;
229 break;
230 case SEC_31_ID:
231 device_set_desc(dev, "Freescale Security Engine 3.1");
232 sc->sc_version = 3;
233 break;
234 default:
235 device_printf(dev, "unknown SEC ID 0x%016llx!\n", id);
236 return (ENXIO);
237 }
238
239 return (0);
240 }
241
242 static int
243 sec_attach(device_t dev)
244 {
245 struct sec_softc *sc;
246 struct sec_hw_lt *lt;
247 int error = 0;
248 int i;
249
250 sc = device_get_softc(dev);
251 sc->sc_dev = dev;
252 sc->sc_blocked = 0;
253 sc->sc_shutdown = 0;
254
255 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
256 if (sc->sc_cid < 0) {
257 device_printf(dev, "could not get crypto driver ID!\n");
258 return (ENXIO);
259 }
260
261 /* Init locks */
262 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
263 "SEC Controller lock", MTX_DEF);
264 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
265 "SEC Descriptors lock", MTX_DEF);
266 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
267 "SEC Sessions lock", MTX_DEF);
268
269 /* Allocate I/O memory for SEC registers */
270 sc->sc_rrid = 0;
271 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
272 RF_ACTIVE);
273
274 if (sc->sc_rres == NULL) {
275 device_printf(dev, "could not allocate I/O memory!\n");
276 goto fail1;
277 }
278
279 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
280 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
281
282 /* Setup interrupts */
283 sc->sc_pri_irid = 0;
284 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
285 &sc->sc_pri_irid, sec_primary_intr, "primary");
286
287 if (error)
288 goto fail2;
289
290
291 if (sc->sc_version == 3) {
292 sc->sc_sec_irid = 1;
293 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
294 &sc->sc_sec_irid, sec_secondary_intr, "secondary");
295
296 if (error)
297 goto fail3;
298 }
299
300 /* Alloc DMA memory for descriptors and link tables */
301 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
302 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
303
304 if (error)
305 goto fail4;
306
307 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
308 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
309
310 if (error)
311 goto fail5;
312
313 /* Fill in descriptors and link tables */
314 for (i = 0; i < SEC_DESCRIPTORS; i++) {
315 sc->sc_desc[i].sd_desc =
316 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
317 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
318 (i * sizeof(struct sec_hw_desc));
319 }
320
321 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
322 sc->sc_lt[i].sl_lt =
323 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
324 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
325 (i * sizeof(struct sec_hw_lt));
326 }
327
328 /* Last entry in link table is used to create a circle */
329 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
330 lt->shl_length = 0;
331 lt->shl_r = 0;
332 lt->shl_n = 1;
333 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
334
335 /* Init descriptor and link table queues pointers */
336 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
337 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
338 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
339 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
340 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
341 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
342 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
343 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
344
345 /* Create masks for fast checks */
346 sc->sc_int_error_mask = 0;
347 for (i = 0; i < SEC_CHANNELS; i++)
348 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
349
350 switch (sc->sc_version) {
351 case 2:
352 sc->sc_channel_idle_mask =
353 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
354 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
355 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
356 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
357 break;
358 case 3:
359 sc->sc_channel_idle_mask =
360 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
361 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
362 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
363 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
364 break;
365 }
366
367 /* Init hardware */
368 error = sec_init(sc);
369
370 if (error)
371 goto fail6;
372
373 /* Register in OCF (AESU) */
374 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
375
376 /* Register in OCF (DEU) */
377 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
378 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
379
380 /* Register in OCF (MDEU) */
381 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
382 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
383 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
384 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
385 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
386 if (sc->sc_version >= 3) {
387 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
388 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
389 }
390
391 return (0);
392
393 fail6:
394 sec_free_dma_mem(&(sc->sc_lt_dmem));
395 fail5:
396 sec_free_dma_mem(&(sc->sc_desc_dmem));
397 fail4:
398 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
399 sc->sc_sec_irid, "secondary");
400 fail3:
401 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
402 sc->sc_pri_irid, "primary");
403 fail2:
404 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
405 fail1:
406 mtx_destroy(&sc->sc_controller_lock);
407 mtx_destroy(&sc->sc_descriptors_lock);
408 mtx_destroy(&sc->sc_sessions_lock);
409
410 return (ENXIO);
411 }
412
413 static int
414 sec_detach(device_t dev)
415 {
416 struct sec_softc *sc = device_get_softc(dev);
417 int i, error, timeout = SEC_TIMEOUT;
418
419 /* Prepare driver to shutdown */
420 SEC_LOCK(sc, descriptors);
421 sc->sc_shutdown = 1;
422 SEC_UNLOCK(sc, descriptors);
423
424 /* Wait until all queued processing finishes */
425 while (1) {
426 SEC_LOCK(sc, descriptors);
427 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
428 SEC_UNLOCK(sc, descriptors);
429
430 if (i == 0)
431 break;
432
433 if (timeout < 0) {
434 device_printf(dev, "queue flush timeout!\n");
435
436 /* DMA can be still active - stop it */
437 for (i = 0; i < SEC_CHANNELS; i++)
438 sec_channel_reset(sc, i, 1);
439
440 break;
441 }
442
443 timeout -= 1000;
444 DELAY(1000);
445 }
446
447 /* Disable interrupts */
448 SEC_WRITE(sc, SEC_IER, 0);
449
450 /* Unregister from OCF */
451 crypto_unregister_all(sc->sc_cid);
452
453 /* Free DMA memory */
454 for (i = 0; i < SEC_DESCRIPTORS; i++)
455 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
456
457 sec_free_dma_mem(&(sc->sc_lt_dmem));
458 sec_free_dma_mem(&(sc->sc_desc_dmem));
459
460 /* Release interrupts */
461 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
462 sc->sc_pri_irid, "primary");
463 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
464 sc->sc_sec_irid, "secondary");
465
466 /* Release memory */
467 if (sc->sc_rres) {
468 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
469 sc->sc_rres);
470 if (error)
471 device_printf(dev, "bus_release_resource() failed for"
472 " I/O memory, error %d\n", error);
473
474 sc->sc_rres = NULL;
475 }
476
477 mtx_destroy(&sc->sc_controller_lock);
478 mtx_destroy(&sc->sc_descriptors_lock);
479 mtx_destroy(&sc->sc_sessions_lock);
480
481 return (0);
482 }
483
484 static int
485 sec_suspend(device_t dev)
486 {
487
488 return (0);
489 }
490
491 static int
492 sec_resume(device_t dev)
493 {
494
495 return (0);
496 }
497
498 static int
499 sec_shutdown(device_t dev)
500 {
501
502 return (0);
503 }
504
505 static int
506 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
507 int *irid, driver_intr_t handler, const char *iname)
508 {
509 int error;
510
511 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
512 RF_ACTIVE);
513
514 if ((*ires) == NULL) {
515 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
516 return (ENXIO);
517 }
518
519 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
520 NULL, handler, sc, ihand);
521
522 if (error) {
523 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
524 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
525 device_printf(sc->sc_dev, "could not release %s IRQ\n",
526 iname);
527
528 (*ires) = NULL;
529 return (error);
530 }
531
532 return (0);
533 }
534
535 static void
536 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
537 int irid, const char *iname)
538 {
539 int error;
540
541 if (ires == NULL)
542 return;
543
544 error = bus_teardown_intr(sc->sc_dev, ires, ihand);
545 if (error)
546 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
547 " IRQ, error %d\n", iname, error);
548
549 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
550 if (error)
551 device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
552 " IRQ, error %d\n", iname, error);
553 }
554
555 static void
556 sec_primary_intr(void *arg)
557 {
558 struct sec_softc *sc = arg;
559 struct sec_desc *desc;
560 uint64_t isr;
561 int i, wakeup = 0;
562
563 SEC_LOCK(sc, controller);
564
565 /* Check for errors */
566 isr = SEC_READ(sc, SEC_ISR);
567 if (isr & sc->sc_int_error_mask) {
568 /* Check each channel for error */
569 for (i = 0; i < SEC_CHANNELS; i++) {
570 if ((isr & SEC_INT_CH_ERR(i)) == 0)
571 continue;
572
573 device_printf(sc->sc_dev,
574 "I/O error on channel %i!\n", i);
575
576 /* Find and mark problematic descriptor */
577 desc = sec_find_desc(sc, SEC_READ(sc,
578 SEC_CHAN_CDPR(i)));
579
580 if (desc != NULL)
581 desc->sd_error = EIO;
582
583 /* Do partial channel reset */
584 sec_channel_reset(sc, i, 0);
585 }
586 }
587
588 /* ACK interrupt */
589 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
590
591 SEC_UNLOCK(sc, controller);
592 SEC_LOCK(sc, descriptors);
593
594 /* Handle processed descriptors */
595 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
596
597 while (SEC_QUEUED_DESC_CNT(sc) > 0) {
598 desc = SEC_GET_QUEUED_DESC(sc);
599
600 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
601 SEC_PUT_BACK_QUEUED_DESC(sc);
602 break;
603 }
604
605 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
606 BUS_DMASYNC_PREWRITE);
607
608 desc->sd_crp->crp_etype = desc->sd_error;
609 crypto_done(desc->sd_crp);
610
611 SEC_DESC_FREE_POINTERS(desc);
612 SEC_DESC_FREE_LT(sc, desc);
613 SEC_DESC_QUEUED2FREE(sc);
614 }
615
616 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
617
618 if (!sc->sc_shutdown) {
619 wakeup = sc->sc_blocked;
620 sc->sc_blocked = 0;
621 }
622
623 SEC_UNLOCK(sc, descriptors);
624
625 /* Enqueue ready descriptors in hardware */
626 sec_enqueue(sc);
627
628 if (wakeup)
629 crypto_unblock(sc->sc_cid, wakeup);
630 }
631
632 static void
633 sec_secondary_intr(void *arg)
634 {
635 struct sec_softc *sc = arg;
636
637 device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
638 sec_primary_intr(arg);
639 }
640
641 static int
642 sec_controller_reset(struct sec_softc *sc)
643 {
644 int timeout = SEC_TIMEOUT;
645
646 /* Reset Controller */
647 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
648
649 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
650 DELAY(1000);
651 timeout -= 1000;
652
653 if (timeout < 0) {
654 device_printf(sc->sc_dev, "timeout while waiting for "
655 "device reset!\n");
656 return (ETIMEDOUT);
657 }
658 }
659
660 return (0);
661 }
662
663 static int
664 sec_channel_reset(struct sec_softc *sc, int channel, int full)
665 {
666 int timeout = SEC_TIMEOUT;
667 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
668 uint64_t reg;
669
670 /* Reset Channel */
671 reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
672 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
673
674 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
675 DELAY(1000);
676 timeout -= 1000;
677
678 if (timeout < 0) {
679 device_printf(sc->sc_dev, "timeout while waiting for "
680 "channel reset!\n");
681 return (ETIMEDOUT);
682 }
683 }
684
685 if (full) {
686 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
687
688 switch(sc->sc_version) {
689 case 2:
690 reg |= SEC_CHAN_CCR_CDWE;
691 break;
692 case 3:
693 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
694 break;
695 }
696
697 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
698 }
699
700 return (0);
701 }
702
703 static int
704 sec_init(struct sec_softc *sc)
705 {
706 uint64_t reg;
707 int error, i;
708
709 /* Reset controller twice to clear all pending interrupts */
710 error = sec_controller_reset(sc);
711 if (error)
712 return (error);
713
714 error = sec_controller_reset(sc);
715 if (error)
716 return (error);
717
718 /* Reset channels */
719 for (i = 0; i < SEC_CHANNELS; i++) {
720 error = sec_channel_reset(sc, i, 1);
721 if (error)
722 return (error);
723 }
724
725 /* Enable Interrupts */
726 reg = SEC_INT_ITO;
727 for (i = 0; i < SEC_CHANNELS; i++)
728 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
729
730 SEC_WRITE(sc, SEC_IER, reg);
731
732 return (error);
733 }
734
735 static void
736 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
737 {
738 struct sec_dma_mem *dma_mem = arg;
739
740 if (error)
741 return;
742
743 KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
744 dma_mem->dma_paddr = segs->ds_addr;
745 }
746
747 static void
748 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
749 int error)
750 {
751 struct sec_desc_map_info *sdmi = arg;
752 struct sec_softc *sc = sdmi->sdmi_sc;
753 struct sec_lt *lt = NULL;
754 bus_addr_t addr;
755 bus_size_t size;
756 int i;
757
758 SEC_LOCK_ASSERT(sc, descriptors);
759
760 if (error)
761 return;
762
763 for (i = 0; i < nseg; i++) {
764 addr = segs[i].ds_addr;
765 size = segs[i].ds_len;
766
767 /* Skip requested offset */
768 if (sdmi->sdmi_offset >= size) {
769 sdmi->sdmi_offset -= size;
770 continue;
771 }
772
773 addr += sdmi->sdmi_offset;
774 size -= sdmi->sdmi_offset;
775 sdmi->sdmi_offset = 0;
776
777 /* Do not link more than requested */
778 if (sdmi->sdmi_size < size)
779 size = sdmi->sdmi_size;
780
781 lt = SEC_ALLOC_LT_ENTRY(sc);
782 lt->sl_lt->shl_length = size;
783 lt->sl_lt->shl_r = 0;
784 lt->sl_lt->shl_n = 0;
785 lt->sl_lt->shl_ptr = addr;
786
787 if (sdmi->sdmi_lt_first == NULL)
788 sdmi->sdmi_lt_first = lt;
789
790 sdmi->sdmi_lt_used += 1;
791
792 if ((sdmi->sdmi_size -= size) == 0)
793 break;
794 }
795
796 sdmi->sdmi_lt_last = lt;
797 }
798
799 static void
800 sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
801 bus_size_t size, int error)
802 {
803
804 sec_dma_map_desc_cb(arg, segs, nseg, error);
805 }
806
807 static int
808 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
809 bus_size_t size)
810 {
811 int error;
812
813 if (dma_mem->dma_vaddr != NULL)
814 return (EBUSY);
815
816 error = bus_dma_tag_create(NULL, /* parent */
817 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
818 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
819 BUS_SPACE_MAXADDR, /* highaddr */
820 NULL, NULL, /* filtfunc, filtfuncarg */
821 size, 1, /* maxsize, nsegments */
822 size, 0, /* maxsegsz, flags */
823 NULL, NULL, /* lockfunc, lockfuncarg */
824 &(dma_mem->dma_tag)); /* dmat */
825
826 if (error) {
827 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
828 " %i!\n", error);
829 goto err1;
830 }
831
832 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
833 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
834
835 if (error) {
836 device_printf(sc->sc_dev, "failed to allocate DMA safe"
837 " memory, error %i!\n", error);
838 goto err2;
839 }
840
841 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
842 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
843 BUS_DMA_NOWAIT);
844
845 if (error) {
846 device_printf(sc->sc_dev, "cannot get address of the DMA"
847 " memory, error %i\n", error);
848 goto err3;
849 }
850
851 dma_mem->dma_is_map = 0;
852 return (0);
853
854 err3:
855 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
856 err2:
857 bus_dma_tag_destroy(dma_mem->dma_tag);
858 err1:
859 dma_mem->dma_vaddr = NULL;
860 return(error);
861 }
862
863 static int
864 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
865 bus_size_t size, int type, struct sec_desc_map_info *sdmi)
866 {
867 int error;
868
869 if (dma_mem->dma_vaddr != NULL)
870 return (EBUSY);
871
872 switch (type) {
873 case SEC_MEMORY:
874 break;
875 case SEC_UIO:
876 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
877 break;
878 case SEC_MBUF:
879 size = m_length((struct mbuf*)mem, NULL);
880 break;
881 default:
882 return (EINVAL);
883 }
884
885 error = bus_dma_tag_create(NULL, /* parent */
886 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
887 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
888 BUS_SPACE_MAXADDR, /* highaddr */
889 NULL, NULL, /* filtfunc, filtfuncarg */
890 size, /* maxsize */
891 SEC_FREE_LT_CNT(sc), /* nsegments */
892 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */
893 NULL, NULL, /* lockfunc, lockfuncarg */
894 &(dma_mem->dma_tag)); /* dmat */
895
896 if (error) {
897 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
898 " %i!\n", error);
899 dma_mem->dma_vaddr = NULL;
900 return (error);
901 }
902
903 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
904
905 if (error) {
906 device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
907 "\n", error);
908 bus_dma_tag_destroy(dma_mem->dma_tag);
909 return (error);
910 }
911
912 switch (type) {
913 case SEC_MEMORY:
914 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
915 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
916 break;
917 case SEC_UIO:
918 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
919 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
920 break;
921 case SEC_MBUF:
922 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
923 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
924 break;
925 }
926
927 if (error) {
928 device_printf(sc->sc_dev, "cannot get address of the DMA"
929 " memory, error %i!\n", error);
930 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
931 bus_dma_tag_destroy(dma_mem->dma_tag);
932 return (error);
933 }
934
935 dma_mem->dma_is_map = 1;
936 dma_mem->dma_vaddr = mem;
937
938 return (0);
939 }
940
941 static void
942 sec_free_dma_mem(struct sec_dma_mem *dma_mem)
943 {
944
945 /* Check for double free */
946 if (dma_mem->dma_vaddr == NULL)
947 return;
948
949 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
950
951 if (dma_mem->dma_is_map)
952 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
953 else
954 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
955 dma_mem->dma_map);
956
957 bus_dma_tag_destroy(dma_mem->dma_tag);
958 dma_mem->dma_vaddr = NULL;
959 }
960
961 static int
962 sec_eu_channel(struct sec_softc *sc, int eu)
963 {
964 uint64_t reg;
965 int channel = 0;
966
967 SEC_LOCK_ASSERT(sc, controller);
968
969 reg = SEC_READ(sc, SEC_EUASR);
970
971 switch (eu) {
972 case SEC_EU_AFEU:
973 channel = SEC_EUASR_AFEU(reg);
974 break;
975 case SEC_EU_DEU:
976 channel = SEC_EUASR_DEU(reg);
977 break;
978 case SEC_EU_MDEU_A:
979 case SEC_EU_MDEU_B:
980 channel = SEC_EUASR_MDEU(reg);
981 break;
982 case SEC_EU_RNGU:
983 channel = SEC_EUASR_RNGU(reg);
984 break;
985 case SEC_EU_PKEU:
986 channel = SEC_EUASR_PKEU(reg);
987 break;
988 case SEC_EU_AESU:
989 channel = SEC_EUASR_AESU(reg);
990 break;
991 case SEC_EU_KEU:
992 channel = SEC_EUASR_KEU(reg);
993 break;
994 case SEC_EU_CRCU:
995 channel = SEC_EUASR_CRCU(reg);
996 break;
997 }
998
999 return (channel - 1);
1000 }
1001
1002 static int
1003 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
1004 {
1005 u_int fflvl = SEC_MAX_FIFO_LEVEL;
1006 uint64_t reg;
1007 int i;
1008
1009 SEC_LOCK_ASSERT(sc, controller);
1010
1011 /* Find free channel if have not got one */
1012 if (channel < 0) {
1013 for (i = 0; i < SEC_CHANNELS; i++) {
1014 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
1015
1016 if ((reg & sc->sc_channel_idle_mask) == 0) {
1017 channel = i;
1018 break;
1019 }
1020 }
1021 }
1022
1023 /* There is no free channel */
1024 if (channel < 0)
1025 return (-1);
1026
1027 /* Check FIFO level on selected channel */
1028 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
1029
1030 switch(sc->sc_version) {
1031 case 2:
1032 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
1033 break;
1034 case 3:
1035 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
1036 break;
1037 }
1038
1039 if (fflvl >= SEC_MAX_FIFO_LEVEL)
1040 return (-1);
1041
1042 /* Enqueue descriptor in channel */
1043 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
1044
1045 return (channel);
1046 }
1047
1048 static void
1049 sec_enqueue(struct sec_softc *sc)
1050 {
1051 struct sec_desc *desc;
1052 int ch0, ch1;
1053
1054 SEC_LOCK(sc, descriptors);
1055 SEC_LOCK(sc, controller);
1056
1057 while (SEC_READY_DESC_CNT(sc) > 0) {
1058 desc = SEC_GET_READY_DESC(sc);
1059
1060 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
1061 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
1062
1063 /*
1064 * Both EU are used by the same channel.
1065 * Enqueue descriptor in channel used by busy EUs.
1066 */
1067 if (ch0 >= 0 && ch0 == ch1) {
1068 if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
1069 SEC_DESC_READY2QUEUED(sc);
1070 continue;
1071 }
1072 }
1073
1074 /*
1075 * Only one EU is free.
1076 * Enqueue descriptor in channel used by busy EU.
1077 */
1078 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
1079 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
1080 >= 0) {
1081 SEC_DESC_READY2QUEUED(sc);
1082 continue;
1083 }
1084 }
1085
1086 /*
1087 * Both EU are free.
1088 * Enqueue descriptor in first free channel.
1089 */
1090 if (ch0 < 0 && ch1 < 0) {
1091 if (sec_enqueue_desc(sc, desc, -1) >= 0) {
1092 SEC_DESC_READY2QUEUED(sc);
1093 continue;
1094 }
1095 }
1096
1097 /* Current descriptor can not be queued at the moment */
1098 SEC_PUT_BACK_READY_DESC(sc);
1099 break;
1100 }
1101
1102 SEC_UNLOCK(sc, controller);
1103 SEC_UNLOCK(sc, descriptors);
1104 }
1105
1106 static struct sec_desc *
1107 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
1108 {
1109 struct sec_desc *desc = NULL;
1110 int i;
1111
1112 SEC_LOCK_ASSERT(sc, descriptors);
1113
1114 for (i = 0; i < SEC_CHANNELS; i++) {
1115 if (sc->sc_desc[i].sd_desc_paddr == paddr) {
1116 desc = &(sc->sc_desc[i]);
1117 break;
1118 }
1119 }
1120
1121 return (desc);
1122 }
1123
1124 static int
1125 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
1126 bus_addr_t data, bus_size_t dsize)
1127 {
1128 struct sec_hw_desc_ptr *ptr;
1129
1130 SEC_LOCK_ASSERT(sc, descriptors);
1131
1132 ptr = &(desc->sd_desc->shd_pointer[n]);
1133 ptr->shdp_length = dsize;
1134 ptr->shdp_extent = 0;
1135 ptr->shdp_j = 0;
1136 ptr->shdp_ptr = data;
1137
1138 return (0);
1139 }
1140
1141 static int
1142 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
1143 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
1144 {
1145 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
1146 struct sec_hw_desc_ptr *ptr;
1147 int error;
1148
1149 SEC_LOCK_ASSERT(sc, descriptors);
1150
1151 /* For flat memory map only requested region */
1152 if (dtype == SEC_MEMORY) {
1153 data = (uint8_t*)(data) + doffset;
1154 sdmi.sdmi_offset = 0;
1155 }
1156
1157 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
1158 dtype, &sdmi);
1159
1160 if (error)
1161 return (error);
1162
1163 sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
1164 desc->sd_lt_used += sdmi.sdmi_lt_used;
1165
1166 ptr = &(desc->sd_desc->shd_pointer[n]);
1167 ptr->shdp_length = dsize;
1168 ptr->shdp_extent = 0;
1169 ptr->shdp_j = 1;
1170 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
1171
1172 return (0);
1173 }
1174
1175 static int
1176 sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
1177 struct cryptoini **mac)
1178 {
1179 struct cryptoini *e, *m;
1180
1181 e = cri;
1182 m = cri->cri_next;
1183
1184 /* We can haldle only two operations */
1185 if (m && m->cri_next)
1186 return (EINVAL);
1187
1188 if (sec_mdeu_can_handle(e->cri_alg)) {
1189 cri = m;
1190 m = e;
1191 e = cri;
1192 }
1193
1194 if (m && !sec_mdeu_can_handle(m->cri_alg))
1195 return (EINVAL);
1196
1197 *enc = e;
1198 *mac = m;
1199
1200 return (0);
1201 }
1202
1203 static int
1204 sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
1205 struct cryptodesc **mac)
1206 {
1207 struct cryptodesc *e, *m, *t;
1208
1209 e = crp->crp_desc;
1210 m = e->crd_next;
1211
1212 /* We can haldle only two operations */
1213 if (m && m->crd_next)
1214 return (EINVAL);
1215
1216 if (sec_mdeu_can_handle(e->crd_alg)) {
1217 t = m;
1218 m = e;
1219 e = t;
1220 }
1221
1222 if (m && !sec_mdeu_can_handle(m->crd_alg))
1223 return (EINVAL);
1224
1225 *enc = e;
1226 *mac = m;
1227
1228 return (0);
1229 }
1230
1231 static int
1232 sec_alloc_session(struct sec_softc *sc)
1233 {
1234 struct sec_session *ses = NULL;
1235 int sid = -1;
1236 u_int i;
1237
1238 SEC_LOCK(sc, sessions);
1239
1240 for (i = 0; i < SEC_MAX_SESSIONS; i++) {
1241 if (sc->sc_sessions[i].ss_used == 0) {
1242 ses = &(sc->sc_sessions[i]);
1243 ses->ss_used = 1;
1244 ses->ss_ivlen = 0;
1245 ses->ss_klen = 0;
1246 ses->ss_mklen = 0;
1247 sid = i;
1248 break;
1249 }
1250 }
1251
1252 SEC_UNLOCK(sc, sessions);
1253
1254 return (sid);
1255 }
1256
1257 static struct sec_session *
1258 sec_get_session(struct sec_softc *sc, u_int sid)
1259 {
1260 struct sec_session *ses;
1261
1262 if (sid >= SEC_MAX_SESSIONS)
1263 return (NULL);
1264
1265 SEC_LOCK(sc, sessions);
1266
1267 ses = &(sc->sc_sessions[sid]);
1268
1269 if (ses->ss_used == 0)
1270 ses = NULL;
1271
1272 SEC_UNLOCK(sc, sessions);
1273
1274 return (ses);
1275 }
1276
1277 static int
1278 sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
1279 {
1280 struct sec_softc *sc = device_get_softc(dev);
1281 struct sec_eu_methods *eu = sec_eus;
1282 struct cryptoini *enc = NULL;
1283 struct cryptoini *mac = NULL;
1284 struct sec_session *ses;
1285 int error = -1;
1286 int sid;
1287
1288 error = sec_split_cri(cri, &enc, &mac);
1289 if (error)
1290 return (error);
1291
1292 /* Check key lengths */
1293 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
1294 return (E2BIG);
1295
1296 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
1297 return (E2BIG);
1298
1299 /* Only SEC 3.0 supports digests larger than 256 bits */
1300 if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
1301 return (E2BIG);
1302
1303 sid = sec_alloc_session(sc);
1304 if (sid < 0)
1305 return (ENOMEM);
1306
1307 ses = sec_get_session(sc, sid);
1308
1309 /* Find EU for this session */
1310 while (eu->sem_make_desc != NULL) {
1311 error = eu->sem_newsession(sc, ses, enc, mac);
1312 if (error >= 0)
1313 break;
1314
1315 eu++;
1316 }
1317
1318 /* If not found, return EINVAL */
1319 if (error < 0) {
1320 sec_free_session(sc, ses);
1321 return (EINVAL);
1322 }
1323
1324 /* Save cipher key */
1325 if (enc && enc->cri_key) {
1326 ses->ss_klen = enc->cri_klen / 8;
1327 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
1328 }
1329
1330 /* Save digest key */
1331 if (mac && mac->cri_key) {
1332 ses->ss_mklen = mac->cri_klen / 8;
1333 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
1334 }
1335
1336 ses->ss_eu = eu;
1337 *sidp = sid;
1338
1339 return (0);
1340 }
1341
1342 static int
1343 sec_freesession(device_t dev, uint64_t tid)
1344 {
1345 struct sec_softc *sc = device_get_softc(dev);
1346 struct sec_session *ses;
1347 int error = 0;
1348
1349 ses = sec_get_session(sc, CRYPTO_SESID2LID(tid));
1350 if (ses == NULL)
1351 return (EINVAL);
1352
1353 sec_free_session(sc, ses);
1354
1355 return (error);
1356 }
1357
1358 static int
1359 sec_process(device_t dev, struct cryptop *crp, int hint)
1360 {
1361 struct sec_softc *sc = device_get_softc(dev);
1362 struct sec_desc *desc = NULL;
1363 struct cryptodesc *mac, *enc;
1364 struct sec_session *ses;
1365 int buftype, error = 0;
1366
1367 /* Check Session ID */
1368 ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid));
1369 if (ses == NULL) {
1370 crp->crp_etype = EINVAL;
1371 crypto_done(crp);
1372 return (0);
1373 }
1374
1375 /* Check for input length */
1376 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
1377 crp->crp_etype = E2BIG;
1378 crypto_done(crp);
1379 return (0);
1380 }
1381
1382 /* Get descriptors */
1383 if (sec_split_crp(crp, &enc, &mac)) {
1384 crp->crp_etype = EINVAL;
1385 crypto_done(crp);
1386 return (0);
1387 }
1388
1389 SEC_LOCK(sc, descriptors);
1390 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1391
1392 /* Block driver if there is no free descriptors or we are going down */
1393 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
1394 sc->sc_blocked |= CRYPTO_SYMQ;
1395 SEC_UNLOCK(sc, descriptors);
1396 return (ERESTART);
1397 }
1398
1399 /* Prepare descriptor */
1400 desc = SEC_GET_FREE_DESC(sc);
1401 desc->sd_lt_used = 0;
1402 desc->sd_error = 0;
1403 desc->sd_crp = crp;
1404
1405 if (crp->crp_flags & CRYPTO_F_IOV)
1406 buftype = SEC_UIO;
1407 else if (crp->crp_flags & CRYPTO_F_IMBUF)
1408 buftype = SEC_MBUF;
1409 else
1410 buftype = SEC_MEMORY;
1411
1412 if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
1413 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1414 memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
1415 ses->ss_ivlen);
1416 else
1417 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
1418
1419 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
1420 crypto_copyback(crp->crp_flags, crp->crp_buf,
1421 enc->crd_inject, ses->ss_ivlen,
1422 desc->sd_desc->shd_iv);
1423 } else if (enc) {
1424 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1425 memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
1426 ses->ss_ivlen);
1427 else
1428 crypto_copydata(crp->crp_flags, crp->crp_buf,
1429 enc->crd_inject, ses->ss_ivlen,
1430 desc->sd_desc->shd_iv);
1431 }
1432
1433 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
1434 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
1435 ses->ss_klen = enc->crd_klen / 8;
1436 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
1437 } else
1438 error = E2BIG;
1439 }
1440
1441 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
1442 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
1443 ses->ss_mklen = mac->crd_klen / 8;
1444 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
1445 } else
1446 error = E2BIG;
1447 }
1448
1449 if (!error) {
1450 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
1451 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
1452
1453 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
1454 }
1455
1456 if (error) {
1457 SEC_DESC_FREE_POINTERS(desc);
1458 SEC_DESC_PUT_BACK_LT(sc, desc);
1459 SEC_PUT_BACK_FREE_DESC(sc);
1460 SEC_UNLOCK(sc, descriptors);
1461 crp->crp_etype = error;
1462 crypto_done(crp);
1463 return (0);
1464 }
1465
1466 /*
1467 * Skip DONE interrupt if this is not last request in burst, but only
1468 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
1469 * signaling on each descriptor.
1470 */
1471 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
1472 desc->sd_desc->shd_dn = 0;
1473 else
1474 desc->sd_desc->shd_dn = 1;
1475
1476 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1477 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
1478 BUS_DMASYNC_POSTWRITE);
1479 SEC_DESC_FREE2READY(sc);
1480 SEC_UNLOCK(sc, descriptors);
1481
1482 /* Enqueue ready descriptors in hardware */
1483 sec_enqueue(sc);
1484
1485 return (0);
1486 }
1487
1488 static int
1489 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
1490 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
1491 int buftype)
1492 {
1493 struct sec_hw_desc *hd = desc->sd_desc;
1494 int error;
1495
1496 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1497 hd->shd_eu_sel1 = SEC_EU_NONE;
1498 hd->shd_mode1 = 0;
1499
1500 /* Pointer 0: NULL */
1501 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1502 if (error)
1503 return (error);
1504
1505 /* Pointer 1: IV IN */
1506 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
1507 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
1508 if (error)
1509 return (error);
1510
1511 /* Pointer 2: Cipher Key */
1512 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1513 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
1514 if (error)
1515 return (error);
1516
1517 /* Pointer 3: Data IN */
1518 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
1519 enc->crd_len, buftype);
1520 if (error)
1521 return (error);
1522
1523 /* Pointer 4: Data OUT */
1524 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
1525 enc->crd_len, buftype);
1526 if (error)
1527 return (error);
1528
1529 /* Pointer 5: IV OUT (Not used: NULL) */
1530 error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
1531 if (error)
1532 return (error);
1533
1534 /* Pointer 6: NULL */
1535 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1536
1537 return (error);
1538 }
1539
1540 static int
1541 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
1542 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
1543 struct cryptodesc *mac, int buftype)
1544 {
1545 struct sec_hw_desc *hd = desc->sd_desc;
1546 u_int eu, mode, hashlen;
1547 int error;
1548
1549 if (mac->crd_len < enc->crd_len)
1550 return (EINVAL);
1551
1552 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
1553 return (EINVAL);
1554
1555 error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
1556 if (error)
1557 return (error);
1558
1559 hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
1560 hd->shd_eu_sel1 = eu;
1561 hd->shd_mode1 = mode;
1562
1563 /* Pointer 0: HMAC Key */
1564 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
1565 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
1566 if (error)
1567 return (error);
1568
1569 /* Pointer 1: HMAC-Only Data IN */
1570 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
1571 mac->crd_len - enc->crd_len, buftype);
1572 if (error)
1573 return (error);
1574
1575 /* Pointer 2: Cipher Key */
1576 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1577 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
1578 if (error)
1579 return (error);
1580
1581 /* Pointer 3: IV IN */
1582 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
1583 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
1584 if (error)
1585 return (error);
1586
1587 /* Pointer 4: Data IN */
1588 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
1589 enc->crd_len, buftype);
1590 if (error)
1591 return (error);
1592
1593 /* Pointer 5: Data OUT */
1594 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
1595 enc->crd_len, buftype);
1596 if (error)
1597 return (error);
1598
1599 /* Pointer 6: HMAC OUT */
1600 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
1601 hashlen, buftype);
1602
1603 return (error);
1604 }
1605
1606 /* AESU */
1607
1608 static int
1609 sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
1610 struct cryptoini *enc, struct cryptoini *mac)
1611 {
1612
1613 if (enc == NULL)
1614 return (-1);
1615
1616 if (enc->cri_alg != CRYPTO_AES_CBC)
1617 return (-1);
1618
1619 ses->ss_ivlen = AES_BLOCK_LEN;
1620
1621 return (0);
1622 }
1623
1624 static int
1625 sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1626 struct sec_desc *desc, struct cryptop *crp, int buftype)
1627 {
1628 struct sec_hw_desc *hd = desc->sd_desc;
1629 struct cryptodesc *enc, *mac;
1630 int error;
1631
1632 error = sec_split_crp(crp, &enc, &mac);
1633 if (error)
1634 return (error);
1635
1636 if (!enc)
1637 return (EINVAL);
1638
1639 hd->shd_eu_sel0 = SEC_EU_AESU;
1640 hd->shd_mode0 = SEC_AESU_MODE_CBC;
1641
1642 if (enc->crd_alg != CRYPTO_AES_CBC)
1643 return (EINVAL);
1644
1645 if (enc->crd_flags & CRD_F_ENCRYPT) {
1646 hd->shd_mode0 |= SEC_AESU_MODE_ED;
1647 hd->shd_dir = 0;
1648 } else
1649 hd->shd_dir = 1;
1650
1651 if (mac)
1652 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
1653 buftype);
1654 else
1655 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
1656 buftype);
1657
1658 return (error);
1659 }
1660
1661 /* DEU */
1662
1663 static int
1664 sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
1665 struct cryptoini *enc, struct cryptoini *mac)
1666 {
1667
1668 if (enc == NULL)
1669 return (-1);
1670
1671 switch (enc->cri_alg) {
1672 case CRYPTO_DES_CBC:
1673 case CRYPTO_3DES_CBC:
1674 break;
1675 default:
1676 return (-1);
1677 }
1678
1679 ses->ss_ivlen = DES_BLOCK_LEN;
1680
1681 return (0);
1682 }
1683
1684 static int
1685 sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1686 struct sec_desc *desc, struct cryptop *crp, int buftype)
1687 {
1688 struct sec_hw_desc *hd = desc->sd_desc;
1689 struct cryptodesc *enc, *mac;
1690 int error;
1691
1692 error = sec_split_crp(crp, &enc, &mac);
1693 if (error)
1694 return (error);
1695
1696 if (!enc)
1697 return (EINVAL);
1698
1699 hd->shd_eu_sel0 = SEC_EU_DEU;
1700 hd->shd_mode0 = SEC_DEU_MODE_CBC;
1701
1702 switch (enc->crd_alg) {
1703 case CRYPTO_3DES_CBC:
1704 hd->shd_mode0 |= SEC_DEU_MODE_TS;
1705 break;
1706 case CRYPTO_DES_CBC:
1707 break;
1708 default:
1709 return (EINVAL);
1710 }
1711
1712 if (enc->crd_flags & CRD_F_ENCRYPT) {
1713 hd->shd_mode0 |= SEC_DEU_MODE_ED;
1714 hd->shd_dir = 0;
1715 } else
1716 hd->shd_dir = 1;
1717
1718 if (mac)
1719 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
1720 buftype);
1721 else
1722 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
1723 buftype);
1724
1725 return (error);
1726 }
1727
1728 /* MDEU */
1729
1730 static int
1731 sec_mdeu_can_handle(u_int alg)
1732 {
1733 switch (alg) {
1734 case CRYPTO_MD5:
1735 case CRYPTO_SHA1:
1736 case CRYPTO_MD5_HMAC:
1737 case CRYPTO_SHA1_HMAC:
1738 case CRYPTO_SHA2_256_HMAC:
1739 case CRYPTO_SHA2_384_HMAC:
1740 case CRYPTO_SHA2_512_HMAC:
1741 return (1);
1742 default:
1743 return (0);
1744 }
1745 }
1746
1747 static int
1748 sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
1749 {
1750
1751 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
1752 *eu = SEC_EU_NONE;
1753
1754 switch (crd->crd_alg) {
1755 case CRYPTO_MD5_HMAC:
1756 *mode |= SEC_MDEU_MODE_HMAC;
1757 /* FALLTHROUGH */
1758 case CRYPTO_MD5:
1759 *eu = SEC_EU_MDEU_A;
1760 *mode |= SEC_MDEU_MODE_MD5;
1761 *hashlen = MD5_HASH_LEN;
1762 break;
1763 case CRYPTO_SHA1_HMAC:
1764 *mode |= SEC_MDEU_MODE_HMAC;
1765 /* FALLTHROUGH */
1766 case CRYPTO_SHA1:
1767 *eu = SEC_EU_MDEU_A;
1768 *mode |= SEC_MDEU_MODE_SHA1;
1769 *hashlen = SHA1_HASH_LEN;
1770 break;
1771 case CRYPTO_SHA2_256_HMAC:
1772 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
1773 *eu = SEC_EU_MDEU_A;
1774 break;
1775 case CRYPTO_SHA2_384_HMAC:
1776 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
1777 *eu = SEC_EU_MDEU_B;
1778 break;
1779 case CRYPTO_SHA2_512_HMAC:
1780 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
1781 *eu = SEC_EU_MDEU_B;
1782 break;
1783 default:
1784 return (EINVAL);
1785 }
1786
1787 if (*mode & SEC_MDEU_MODE_HMAC)
1788 *hashlen = SEC_HMAC_HASH_LEN;
1789
1790 return (0);
1791 }
1792
1793 static int
1794 sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
1795 struct cryptoini *enc, struct cryptoini *mac)
1796 {
1797
1798 if (mac && sec_mdeu_can_handle(mac->cri_alg))
1799 return (0);
1800
1801 return (-1);
1802 }
1803
1804 static int
1805 sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
1806 struct sec_desc *desc, struct cryptop *crp, int buftype)
1807 {
1808 struct cryptodesc *enc, *mac;
1809 struct sec_hw_desc *hd = desc->sd_desc;
1810 u_int eu, mode, hashlen;
1811 int error;
1812
1813 error = sec_split_crp(crp, &enc, &mac);
1814 if (error)
1815 return (error);
1816
1817 if (enc)
1818 return (EINVAL);
1819
1820 error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
1821 if (error)
1822 return (error);
1823
1824 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1825 hd->shd_eu_sel0 = eu;
1826 hd->shd_mode0 = mode;
1827 hd->shd_eu_sel1 = SEC_EU_NONE;
1828 hd->shd_mode1 = 0;
1829
1830 /* Pointer 0: NULL */
1831 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1832 if (error)
1833 return (error);
1834
1835 /* Pointer 1: Context In (Not used: NULL) */
1836 error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
1837 if (error)
1838 return (error);
1839
1840 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */
1841 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
1842 error = sec_make_pointer_direct(sc, desc, 2,
1843 desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
1844 shd_mkey), ses->ss_mklen);
1845 else
1846 error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
1847
1848 if (error)
1849 return (error);
1850
1851 /* Pointer 3: Input Data */
1852 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
1853 mac->crd_len, buftype);
1854 if (error)
1855 return (error);
1856
1857 /* Pointer 4: NULL */
1858 error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
1859 if (error)
1860 return (error);
1861
1862 /* Pointer 5: Hash out */
1863 error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
1864 mac->crd_inject, hashlen, buftype);
1865 if (error)
1866 return (error);
1867
1868 /* Pointer 6: NULL */
1869 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1870
1871 return (0);
1872 }
Cache object: 3825c608ce2634480bd1207a33e5b7b0
|