1 /*
2 * vim:sw=4 ts=8
3 */
4 /*-
5 * SPDX-License-Identifier: BSD-4-Clause
6 *
7 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
8 *
9 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
10 * reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions are met:
14 * 1. Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation
18 * and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by Cavium Networks
22 * 4. Cavium Networks' name may not be used to endorse or promote products
23 * derived from this software without specific prior written permission.
24 *
25 * This Software, including technical data, may be subject to U.S. export
26 * control laws, including the U.S. Export Administration Act and its
27 * associated regulations, and may be subject to export or import regulations
28 * in other countries. You warrant that You will comply strictly in all
29 * respects with all such regulations and acknowledge that you have the
30 * responsibility to obtain licenses to export, re-export or import the
31 * Software.
32 *
33 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
34 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
35 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
36 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
37 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
38 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
39 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
40 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
41 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
42 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
43 */
44 /****************************************************************************/
45
46 #include <sys/cdefs.h>
47 __FBSDID("$FreeBSD: releng/12.0/sys/mips/cavium/cryptocteon/cavium_crypto.c 326259 2017-11-27 15:07:26Z pfg $");
48
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/kernel.h>
52 #include <sys/module.h>
53 #include <sys/malloc.h>
54 #include <sys/uio.h>
55
56 #include <opencrypto/cryptodev.h>
57
58 #include <contrib/octeon-sdk/cvmx.h>
59
60 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
61
62 /****************************************************************************/
63
64 #define IOV_INIT(iov, ptr, idx, len) \
65 do { \
66 (idx) = 0; \
67 (ptr) = (iov)[(idx)].iov_base; \
68 (len) = (iov)[(idx)].iov_len; \
69 } while (0)
70
71 /*
72 * XXX
73 * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
74 * that we could detect overflow before it happens rather than right after,
75 * which is especially bad since there is usually no IOV_CONSUME after the
76 * final read or write.
77 */
78 #define IOV_CONSUME(iov, ptr, idx, len) \
79 do { \
80 if ((len) > sizeof *(ptr)) { \
81 (len) -= sizeof *(ptr); \
82 (ptr)++; \
83 } else { \
84 if ((len) != sizeof *(ptr)) \
85 panic("%s: went past end of iovec.", __func__); \
86 (idx)++; \
87 (ptr) = (iov)[(idx)].iov_base; \
88 (len) = (iov)[(idx)].iov_len; \
89 } \
90 } while (0)
91
92 #define ESP_HEADER_LENGTH 8
93 #define DES_CBC_IV_LENGTH 8
94 #define AES_CBC_IV_LENGTH 16
95 #define ESP_HMAC_LEN 12
96
97 #define ESP_HEADER_LENGTH 8
98 #define DES_CBC_IV_LENGTH 8
99
100 /****************************************************************************/
101
102 #define CVM_LOAD_SHA_UNIT(dat, next) { \
103 if (next == 0) { \
104 next = 1; \
105 CVMX_MT_HSH_DAT (dat, 0); \
106 } else if (next == 1) { \
107 next = 2; \
108 CVMX_MT_HSH_DAT (dat, 1); \
109 } else if (next == 2) { \
110 next = 3; \
111 CVMX_MT_HSH_DAT (dat, 2); \
112 } else if (next == 3) { \
113 next = 4; \
114 CVMX_MT_HSH_DAT (dat, 3); \
115 } else if (next == 4) { \
116 next = 5; \
117 CVMX_MT_HSH_DAT (dat, 4); \
118 } else if (next == 5) { \
119 next = 6; \
120 CVMX_MT_HSH_DAT (dat, 5); \
121 } else if (next == 6) { \
122 next = 7; \
123 CVMX_MT_HSH_DAT (dat, 6); \
124 } else { \
125 CVMX_MT_HSH_STARTSHA (dat); \
126 next = 0; \
127 } \
128 }
129
130 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \
131 if (next == 0) { \
132 CVMX_MT_HSH_DAT (dat1, 0); \
133 CVMX_MT_HSH_DAT (dat2, 1); \
134 next = 2; \
135 } else if (next == 1) { \
136 CVMX_MT_HSH_DAT (dat1, 1); \
137 CVMX_MT_HSH_DAT (dat2, 2); \
138 next = 3; \
139 } else if (next == 2) { \
140 CVMX_MT_HSH_DAT (dat1, 2); \
141 CVMX_MT_HSH_DAT (dat2, 3); \
142 next = 4; \
143 } else if (next == 3) { \
144 CVMX_MT_HSH_DAT (dat1, 3); \
145 CVMX_MT_HSH_DAT (dat2, 4); \
146 next = 5; \
147 } else if (next == 4) { \
148 CVMX_MT_HSH_DAT (dat1, 4); \
149 CVMX_MT_HSH_DAT (dat2, 5); \
150 next = 6; \
151 } else if (next == 5) { \
152 CVMX_MT_HSH_DAT (dat1, 5); \
153 CVMX_MT_HSH_DAT (dat2, 6); \
154 next = 7; \
155 } else if (next == 6) { \
156 CVMX_MT_HSH_DAT (dat1, 6); \
157 CVMX_MT_HSH_STARTSHA (dat2); \
158 next = 0; \
159 } else { \
160 CVMX_MT_HSH_STARTSHA (dat1); \
161 CVMX_MT_HSH_DAT (dat2, 0); \
162 next = 1; \
163 } \
164 }
165
166 /****************************************************************************/
167
168 #define CVM_LOAD_MD5_UNIT(dat, next) { \
169 if (next == 0) { \
170 next = 1; \
171 CVMX_MT_HSH_DAT (dat, 0); \
172 } else if (next == 1) { \
173 next = 2; \
174 CVMX_MT_HSH_DAT (dat, 1); \
175 } else if (next == 2) { \
176 next = 3; \
177 CVMX_MT_HSH_DAT (dat, 2); \
178 } else if (next == 3) { \
179 next = 4; \
180 CVMX_MT_HSH_DAT (dat, 3); \
181 } else if (next == 4) { \
182 next = 5; \
183 CVMX_MT_HSH_DAT (dat, 4); \
184 } else if (next == 5) { \
185 next = 6; \
186 CVMX_MT_HSH_DAT (dat, 5); \
187 } else if (next == 6) { \
188 next = 7; \
189 CVMX_MT_HSH_DAT (dat, 6); \
190 } else { \
191 CVMX_MT_HSH_STARTMD5 (dat); \
192 next = 0; \
193 } \
194 }
195
196 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \
197 if (next == 0) { \
198 CVMX_MT_HSH_DAT (dat1, 0); \
199 CVMX_MT_HSH_DAT (dat2, 1); \
200 next = 2; \
201 } else if (next == 1) { \
202 CVMX_MT_HSH_DAT (dat1, 1); \
203 CVMX_MT_HSH_DAT (dat2, 2); \
204 next = 3; \
205 } else if (next == 2) { \
206 CVMX_MT_HSH_DAT (dat1, 2); \
207 CVMX_MT_HSH_DAT (dat2, 3); \
208 next = 4; \
209 } else if (next == 3) { \
210 CVMX_MT_HSH_DAT (dat1, 3); \
211 CVMX_MT_HSH_DAT (dat2, 4); \
212 next = 5; \
213 } else if (next == 4) { \
214 CVMX_MT_HSH_DAT (dat1, 4); \
215 CVMX_MT_HSH_DAT (dat2, 5); \
216 next = 6; \
217 } else if (next == 5) { \
218 CVMX_MT_HSH_DAT (dat1, 5); \
219 CVMX_MT_HSH_DAT (dat2, 6); \
220 next = 7; \
221 } else if (next == 6) { \
222 CVMX_MT_HSH_DAT (dat1, 6); \
223 CVMX_MT_HSH_STARTMD5 (dat2); \
224 next = 0; \
225 } else { \
226 CVMX_MT_HSH_STARTMD5 (dat1); \
227 CVMX_MT_HSH_DAT (dat2, 0); \
228 next = 1; \
229 } \
230 }
231
232 /****************************************************************************/
233
234 void
235 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
236 {
237 uint8_t hash_key[64];
238 uint64_t *key1;
239 register uint64_t xor1 = 0x3636363636363636ULL;
240 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
241
242 dprintf("%s()\n", __func__);
243
244 memset(hash_key, 0, sizeof(hash_key));
245 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
246 key1 = (uint64_t *) hash_key;
247 if (auth) {
248 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
249 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
250 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
251 } else {
252 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
253 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
254 }
255
256 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
257 key1++;
258 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
259 key1++;
260 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
261 key1++;
262 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
263 key1++;
264 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
265 key1++;
266 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
267 key1++;
268 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
269 key1++;
270 if (auth)
271 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
272 else
273 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
274
275 CVMX_MF_HSH_IV(inner[0], 0);
276 CVMX_MF_HSH_IV(inner[1], 1);
277 if (auth) {
278 inner[2] = 0;
279 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
280 }
281
282 memset(hash_key, 0, sizeof(hash_key));
283 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
284 key1 = (uint64_t *) hash_key;
285 if (auth) {
286 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
287 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
288 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
289 } else {
290 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
291 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
292 }
293
294 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
295 key1++;
296 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
297 key1++;
298 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
299 key1++;
300 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
301 key1++;
302 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
303 key1++;
304 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
305 key1++;
306 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
307 key1++;
308 if (auth)
309 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
310 else
311 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
312
313 CVMX_MF_HSH_IV(outer[0], 0);
314 CVMX_MF_HSH_IV(outer[1], 1);
315 if (auth) {
316 outer[2] = 0;
317 CVMX_MF_HSH_IV(outer[2], 2);
318 }
319 return;
320 }
321
322 /****************************************************************************/
323 /* DES functions */
324
325 int
326 octo_des_cbc_encrypt(
327 struct octo_sess *od,
328 struct iovec *iov, size_t iovcnt, size_t iovlen,
329 int auth_off, int auth_len,
330 int crypt_off, int crypt_len,
331 int icv_off, uint8_t *ivp)
332 {
333 uint64_t *data;
334 int data_i, data_l;
335
336 dprintf("%s()\n", __func__);
337
338 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
339 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
340 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
341 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
342 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
343 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
344 return -EINVAL;
345 }
346
347 IOV_INIT(iov, data, data_i, data_l);
348
349 CVMX_PREFETCH0(ivp);
350 CVMX_PREFETCH0(od->octo_enckey);
351
352
353 /* load 3DES Key */
354 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
355 if (od->octo_encklen == 24) {
356 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
358 } else if (od->octo_encklen == 8) {
359 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
360 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
361 } else {
362 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
363 return -EINVAL;
364 }
365
366 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
367
368 while (crypt_off > 0) {
369 IOV_CONSUME(iov, data, data_i, data_l);
370 crypt_off -= 8;
371 }
372
373 while (crypt_len > 0) {
374 CVMX_MT_3DES_ENC_CBC(*data);
375 CVMX_MF_3DES_RESULT(*data);
376 IOV_CONSUME(iov, data, data_i, data_l);
377 crypt_len -= 8;
378 }
379
380 return 0;
381 }
382
383
384 int
385 octo_des_cbc_decrypt(
386 struct octo_sess *od,
387 struct iovec *iov, size_t iovcnt, size_t iovlen,
388 int auth_off, int auth_len,
389 int crypt_off, int crypt_len,
390 int icv_off, uint8_t *ivp)
391 {
392 uint64_t *data;
393 int data_i, data_l;
394
395 dprintf("%s()\n", __func__);
396
397 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
398 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
399 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
400 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
401 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
402 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
403 return -EINVAL;
404 }
405
406 IOV_INIT(iov, data, data_i, data_l);
407
408 CVMX_PREFETCH0(ivp);
409 CVMX_PREFETCH0(od->octo_enckey);
410
411 /* load 3DES Key */
412 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
413 if (od->octo_encklen == 24) {
414 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
415 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
416 } else if (od->octo_encklen == 8) {
417 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
418 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
419 } else {
420 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
421 return -EINVAL;
422 }
423
424 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
425
426 while (crypt_off > 0) {
427 IOV_CONSUME(iov, data, data_i, data_l);
428 crypt_off -= 8;
429 }
430
431 while (crypt_len > 0) {
432 CVMX_MT_3DES_DEC_CBC(*data);
433 CVMX_MF_3DES_RESULT(*data);
434 IOV_CONSUME(iov, data, data_i, data_l);
435 crypt_len -= 8;
436 }
437
438 return 0;
439 }
440
441 /****************************************************************************/
442 /* AES functions */
443
444 int
445 octo_aes_cbc_encrypt(
446 struct octo_sess *od,
447 struct iovec *iov, size_t iovcnt, size_t iovlen,
448 int auth_off, int auth_len,
449 int crypt_off, int crypt_len,
450 int icv_off, uint8_t *ivp)
451 {
452 uint64_t *data, *pdata;
453 int data_i, data_l;
454
455 dprintf("%s()\n", __func__);
456
457 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
458 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
459 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
460 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
461 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
462 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
463 return -EINVAL;
464 }
465
466 IOV_INIT(iov, data, data_i, data_l);
467
468 CVMX_PREFETCH0(ivp);
469 CVMX_PREFETCH0(od->octo_enckey);
470
471 /* load AES Key */
472 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
473 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
474
475 if (od->octo_encklen == 16) {
476 CVMX_MT_AES_KEY(0x0, 2);
477 CVMX_MT_AES_KEY(0x0, 3);
478 } else if (od->octo_encklen == 24) {
479 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
480 CVMX_MT_AES_KEY(0x0, 3);
481 } else if (od->octo_encklen == 32) {
482 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
483 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
484 } else {
485 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
486 return -EINVAL;
487 }
488 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
489
490 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
491 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
492
493 while (crypt_off > 0) {
494 IOV_CONSUME(iov, data, data_i, data_l);
495 crypt_off -= 8;
496 }
497
498 while (crypt_len > 0) {
499 pdata = data;
500 CVMX_MT_AES_ENC_CBC0(*data);
501 IOV_CONSUME(iov, data, data_i, data_l);
502 CVMX_MT_AES_ENC_CBC1(*data);
503 CVMX_MF_AES_RESULT(*pdata, 0);
504 CVMX_MF_AES_RESULT(*data, 1);
505 IOV_CONSUME(iov, data, data_i, data_l);
506 crypt_len -= 16;
507 }
508
509 return 0;
510 }
511
512
513 int
514 octo_aes_cbc_decrypt(
515 struct octo_sess *od,
516 struct iovec *iov, size_t iovcnt, size_t iovlen,
517 int auth_off, int auth_len,
518 int crypt_off, int crypt_len,
519 int icv_off, uint8_t *ivp)
520 {
521 uint64_t *data, *pdata;
522 int data_i, data_l;
523
524 dprintf("%s()\n", __func__);
525
526 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
527 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
528 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
529 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
530 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
531 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
532 return -EINVAL;
533 }
534
535 IOV_INIT(iov, data, data_i, data_l);
536
537 CVMX_PREFETCH0(ivp);
538 CVMX_PREFETCH0(od->octo_enckey);
539
540 /* load AES Key */
541 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
542 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
543
544 if (od->octo_encklen == 16) {
545 CVMX_MT_AES_KEY(0x0, 2);
546 CVMX_MT_AES_KEY(0x0, 3);
547 } else if (od->octo_encklen == 24) {
548 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
549 CVMX_MT_AES_KEY(0x0, 3);
550 } else if (od->octo_encklen == 32) {
551 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
552 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
553 } else {
554 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
555 return -EINVAL;
556 }
557 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
558
559 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
560 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
561
562 while (crypt_off > 0) {
563 IOV_CONSUME(iov, data, data_i, data_l);
564 crypt_off -= 8;
565 }
566
567 while (crypt_len > 0) {
568 pdata = data;
569 CVMX_MT_AES_DEC_CBC0(*data);
570 IOV_CONSUME(iov, data, data_i, data_l);
571 CVMX_MT_AES_DEC_CBC1(*data);
572 CVMX_MF_AES_RESULT(*pdata, 0);
573 CVMX_MF_AES_RESULT(*data, 1);
574 IOV_CONSUME(iov, data, data_i, data_l);
575 crypt_len -= 16;
576 }
577
578 return 0;
579 }
580
581 /****************************************************************************/
582 /* MD5 */
583
584 int
585 octo_null_md5_encrypt(
586 struct octo_sess *od,
587 struct iovec *iov, size_t iovcnt, size_t iovlen,
588 int auth_off, int auth_len,
589 int crypt_off, int crypt_len,
590 int icv_off, uint8_t *ivp)
591 {
592 int next = 0;
593 uint64_t *data;
594 uint64_t tmp1, tmp2;
595 int data_i, data_l, alen = auth_len;
596
597 dprintf("%s()\n", __func__);
598
599 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
600 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
601 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
602 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
603 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
604 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
605 return -EINVAL;
606 }
607
608 IOV_INIT(iov, data, data_i, data_l);
609
610 /* Load MD5 IV */
611 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
612 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
613
614 while (auth_off > 0) {
615 IOV_CONSUME(iov, data, data_i, data_l);
616 auth_off -= 8;
617 }
618
619 while (auth_len > 0) {
620 CVM_LOAD_MD5_UNIT(*data, next);
621 auth_len -= 8;
622 IOV_CONSUME(iov, data, data_i, data_l);
623 }
624
625 /* finish the hash */
626 CVMX_PREFETCH0(od->octo_hmouter);
627 #if 0
628 if (__predict_false(inplen)) {
629 uint64_t tmp = 0;
630 uint8_t *p = (uint8_t *) & tmp;
631 p[inplen] = 0x80;
632 do {
633 inplen--;
634 p[inplen] = ((uint8_t *) data)[inplen];
635 } while (inplen);
636 CVM_LOAD_MD5_UNIT(tmp, next);
637 } else {
638 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
639 }
640 #else
641 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
642 #endif
643
644 /* Finish Inner hash */
645 while (next != 7) {
646 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
647 }
648 CVMX_ES64(tmp1, ((alen + 64) << 3));
649 CVM_LOAD_MD5_UNIT(tmp1, next);
650
651 /* Get the inner hash of HMAC */
652 CVMX_MF_HSH_IV(tmp1, 0);
653 CVMX_MF_HSH_IV(tmp2, 1);
654
655 /* Initialize hash unit */
656 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
657 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
658
659 CVMX_MT_HSH_DAT(tmp1, 0);
660 CVMX_MT_HSH_DAT(tmp2, 1);
661 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
662 CVMX_MT_HSH_DATZ(3);
663 CVMX_MT_HSH_DATZ(4);
664 CVMX_MT_HSH_DATZ(5);
665 CVMX_MT_HSH_DATZ(6);
666 CVMX_ES64(tmp1, ((64 + 16) << 3));
667 CVMX_MT_HSH_STARTMD5(tmp1);
668
669 /* save the HMAC */
670 IOV_INIT(iov, data, data_i, data_l);
671 while (icv_off > 0) {
672 IOV_CONSUME(iov, data, data_i, data_l);
673 icv_off -= 8;
674 }
675 CVMX_MF_HSH_IV(*data, 0);
676 IOV_CONSUME(iov, data, data_i, data_l);
677 CVMX_MF_HSH_IV(tmp1, 1);
678 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
679
680 return 0;
681 }
682
683 /****************************************************************************/
684 /* SHA1 */
685
686 int
687 octo_null_sha1_encrypt(
688 struct octo_sess *od,
689 struct iovec *iov, size_t iovcnt, size_t iovlen,
690 int auth_off, int auth_len,
691 int crypt_off, int crypt_len,
692 int icv_off, uint8_t *ivp)
693 {
694 int next = 0;
695 uint64_t *data;
696 uint64_t tmp1, tmp2, tmp3;
697 int data_i, data_l, alen = auth_len;
698
699 dprintf("%s()\n", __func__);
700
701 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
702 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
703 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
704 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
705 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
706 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
707 return -EINVAL;
708 }
709
710 IOV_INIT(iov, data, data_i, data_l);
711
712 /* Load SHA1 IV */
713 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
714 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
715 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
716
717 while (auth_off > 0) {
718 IOV_CONSUME(iov, data, data_i, data_l);
719 auth_off -= 8;
720 }
721
722 while (auth_len > 0) {
723 CVM_LOAD_SHA_UNIT(*data, next);
724 auth_len -= 8;
725 IOV_CONSUME(iov, data, data_i, data_l);
726 }
727
728 /* finish the hash */
729 CVMX_PREFETCH0(od->octo_hmouter);
730 #if 0
731 if (__predict_false(inplen)) {
732 uint64_t tmp = 0;
733 uint8_t *p = (uint8_t *) & tmp;
734 p[inplen] = 0x80;
735 do {
736 inplen--;
737 p[inplen] = ((uint8_t *) data)[inplen];
738 } while (inplen);
739 CVM_LOAD_MD5_UNIT(tmp, next);
740 } else {
741 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
742 }
743 #else
744 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
745 #endif
746
747 /* Finish Inner hash */
748 while (next != 7) {
749 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
750 }
751 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
752
753 /* Get the inner hash of HMAC */
754 CVMX_MF_HSH_IV(tmp1, 0);
755 CVMX_MF_HSH_IV(tmp2, 1);
756 tmp3 = 0;
757 CVMX_MF_HSH_IV(tmp3, 2);
758
759 /* Initialize hash unit */
760 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
761 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
762 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
763
764 CVMX_MT_HSH_DAT(tmp1, 0);
765 CVMX_MT_HSH_DAT(tmp2, 1);
766 tmp3 |= 0x0000000080000000;
767 CVMX_MT_HSH_DAT(tmp3, 2);
768 CVMX_MT_HSH_DATZ(3);
769 CVMX_MT_HSH_DATZ(4);
770 CVMX_MT_HSH_DATZ(5);
771 CVMX_MT_HSH_DATZ(6);
772 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
773
774 /* save the HMAC */
775 IOV_INIT(iov, data, data_i, data_l);
776 while (icv_off > 0) {
777 IOV_CONSUME(iov, data, data_i, data_l);
778 icv_off -= 8;
779 }
780 CVMX_MF_HSH_IV(*data, 0);
781 IOV_CONSUME(iov, data, data_i, data_l);
782 CVMX_MF_HSH_IV(tmp1, 1);
783 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
784
785 return 0;
786 }
787
788 /****************************************************************************/
789 /* DES MD5 */
790
791 int
792 octo_des_cbc_md5_encrypt(
793 struct octo_sess *od,
794 struct iovec *iov, size_t iovcnt, size_t iovlen,
795 int auth_off, int auth_len,
796 int crypt_off, int crypt_len,
797 int icv_off, uint8_t *ivp)
798 {
799 int next = 0;
800 union {
801 uint32_t data32[2];
802 uint64_t data64[1];
803 } mydata;
804 uint64_t *data = &mydata.data64[0];
805 uint32_t *data32;
806 uint64_t tmp1, tmp2;
807 int data_i, data_l, alen = auth_len;
808
809 dprintf("%s()\n", __func__);
810
811 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
812 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
813 (crypt_len & 0x7) ||
814 (auth_len & 0x7) ||
815 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
816 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
817 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
818 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
819 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
820 return -EINVAL;
821 }
822
823 IOV_INIT(iov, data32, data_i, data_l);
824
825 CVMX_PREFETCH0(ivp);
826 CVMX_PREFETCH0(od->octo_enckey);
827
828 /* load 3DES Key */
829 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
830 if (od->octo_encklen == 24) {
831 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
832 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
833 } else if (od->octo_encklen == 8) {
834 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
835 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
836 } else {
837 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
838 return -EINVAL;
839 }
840
841 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
842
843 /* Load MD5 IV */
844 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
845 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
846
847 while (crypt_off > 0 && auth_off > 0) {
848 IOV_CONSUME(iov, data32, data_i, data_l);
849 crypt_off -= 4;
850 auth_off -= 4;
851 }
852
853 while (crypt_len > 0 || auth_len > 0) {
854 uint32_t *first = data32;
855 mydata.data32[0] = *first;
856 IOV_CONSUME(iov, data32, data_i, data_l);
857 mydata.data32[1] = *data32;
858 if (crypt_off <= 0) {
859 if (crypt_len > 0) {
860 CVMX_MT_3DES_ENC_CBC(*data);
861 CVMX_MF_3DES_RESULT(*data);
862 crypt_len -= 8;
863 }
864 } else
865 crypt_off -= 8;
866 if (auth_off <= 0) {
867 if (auth_len > 0) {
868 CVM_LOAD_MD5_UNIT(*data, next);
869 auth_len -= 8;
870 }
871 } else
872 auth_off -= 8;
873 *first = mydata.data32[0];
874 *data32 = mydata.data32[1];
875 IOV_CONSUME(iov, data32, data_i, data_l);
876 }
877
878 /* finish the hash */
879 CVMX_PREFETCH0(od->octo_hmouter);
880 #if 0
881 if (__predict_false(inplen)) {
882 uint64_t tmp = 0;
883 uint8_t *p = (uint8_t *) & tmp;
884 p[inplen] = 0x80;
885 do {
886 inplen--;
887 p[inplen] = ((uint8_t *) data)[inplen];
888 } while (inplen);
889 CVM_LOAD_MD5_UNIT(tmp, next);
890 } else {
891 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
892 }
893 #else
894 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
895 #endif
896
897 /* Finish Inner hash */
898 while (next != 7) {
899 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
900 }
901 CVMX_ES64(tmp1, ((alen + 64) << 3));
902 CVM_LOAD_MD5_UNIT(tmp1, next);
903
904 /* Get the inner hash of HMAC */
905 CVMX_MF_HSH_IV(tmp1, 0);
906 CVMX_MF_HSH_IV(tmp2, 1);
907
908 /* Initialize hash unit */
909 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
910 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
911
912 CVMX_MT_HSH_DAT(tmp1, 0);
913 CVMX_MT_HSH_DAT(tmp2, 1);
914 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
915 CVMX_MT_HSH_DATZ(3);
916 CVMX_MT_HSH_DATZ(4);
917 CVMX_MT_HSH_DATZ(5);
918 CVMX_MT_HSH_DATZ(6);
919 CVMX_ES64(tmp1, ((64 + 16) << 3));
920 CVMX_MT_HSH_STARTMD5(tmp1);
921
922 /* save the HMAC */
923 IOV_INIT(iov, data32, data_i, data_l);
924 while (icv_off > 0) {
925 IOV_CONSUME(iov, data32, data_i, data_l);
926 icv_off -= 4;
927 }
928 CVMX_MF_HSH_IV(tmp1, 0);
929 *data32 = (uint32_t) (tmp1 >> 32);
930 IOV_CONSUME(iov, data32, data_i, data_l);
931 *data32 = (uint32_t) tmp1;
932 IOV_CONSUME(iov, data32, data_i, data_l);
933 CVMX_MF_HSH_IV(tmp1, 1);
934 *data32 = (uint32_t) (tmp1 >> 32);
935
936 return 0;
937 }
938
939 int
940 octo_des_cbc_md5_decrypt(
941 struct octo_sess *od,
942 struct iovec *iov, size_t iovcnt, size_t iovlen,
943 int auth_off, int auth_len,
944 int crypt_off, int crypt_len,
945 int icv_off, uint8_t *ivp)
946 {
947 int next = 0;
948 union {
949 uint32_t data32[2];
950 uint64_t data64[1];
951 } mydata;
952 uint64_t *data = &mydata.data64[0];
953 uint32_t *data32;
954 uint64_t tmp1, tmp2;
955 int data_i, data_l, alen = auth_len;
956
957 dprintf("%s()\n", __func__);
958
959 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
960 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
961 (crypt_len & 0x7) ||
962 (auth_len & 0x7) ||
963 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
964 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
965 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
966 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
967 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
968 return -EINVAL;
969 }
970
971 IOV_INIT(iov, data32, data_i, data_l);
972
973 CVMX_PREFETCH0(ivp);
974 CVMX_PREFETCH0(od->octo_enckey);
975
976 /* load 3DES Key */
977 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
978 if (od->octo_encklen == 24) {
979 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
980 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
981 } else if (od->octo_encklen == 8) {
982 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
983 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
984 } else {
985 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
986 return -EINVAL;
987 }
988
989 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
990
991 /* Load MD5 IV */
992 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
993 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
994
995 while (crypt_off > 0 && auth_off > 0) {
996 IOV_CONSUME(iov, data32, data_i, data_l);
997 crypt_off -= 4;
998 auth_off -= 4;
999 }
1000
1001 while (crypt_len > 0 || auth_len > 0) {
1002 uint32_t *first = data32;
1003 mydata.data32[0] = *first;
1004 IOV_CONSUME(iov, data32, data_i, data_l);
1005 mydata.data32[1] = *data32;
1006 if (auth_off <= 0) {
1007 if (auth_len > 0) {
1008 CVM_LOAD_MD5_UNIT(*data, next);
1009 auth_len -= 8;
1010 }
1011 } else
1012 auth_off -= 8;
1013 if (crypt_off <= 0) {
1014 if (crypt_len > 0) {
1015 CVMX_MT_3DES_DEC_CBC(*data);
1016 CVMX_MF_3DES_RESULT(*data);
1017 crypt_len -= 8;
1018 }
1019 } else
1020 crypt_off -= 8;
1021 *first = mydata.data32[0];
1022 *data32 = mydata.data32[1];
1023 IOV_CONSUME(iov, data32, data_i, data_l);
1024 }
1025
1026 /* finish the hash */
1027 CVMX_PREFETCH0(od->octo_hmouter);
1028 #if 0
1029 if (__predict_false(inplen)) {
1030 uint64_t tmp = 0;
1031 uint8_t *p = (uint8_t *) & tmp;
1032 p[inplen] = 0x80;
1033 do {
1034 inplen--;
1035 p[inplen] = ((uint8_t *) data)[inplen];
1036 } while (inplen);
1037 CVM_LOAD_MD5_UNIT(tmp, next);
1038 } else {
1039 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1040 }
1041 #else
1042 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1043 #endif
1044
1045 /* Finish Inner hash */
1046 while (next != 7) {
1047 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1048 }
1049 CVMX_ES64(tmp1, ((alen + 64) << 3));
1050 CVM_LOAD_MD5_UNIT(tmp1, next);
1051
1052 /* Get the inner hash of HMAC */
1053 CVMX_MF_HSH_IV(tmp1, 0);
1054 CVMX_MF_HSH_IV(tmp2, 1);
1055
1056 /* Initialize hash unit */
1057 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1058 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1059
1060 CVMX_MT_HSH_DAT(tmp1, 0);
1061 CVMX_MT_HSH_DAT(tmp2, 1);
1062 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1063 CVMX_MT_HSH_DATZ(3);
1064 CVMX_MT_HSH_DATZ(4);
1065 CVMX_MT_HSH_DATZ(5);
1066 CVMX_MT_HSH_DATZ(6);
1067 CVMX_ES64(tmp1, ((64 + 16) << 3));
1068 CVMX_MT_HSH_STARTMD5(tmp1);
1069
1070 /* save the HMAC */
1071 IOV_INIT(iov, data32, data_i, data_l);
1072 while (icv_off > 0) {
1073 IOV_CONSUME(iov, data32, data_i, data_l);
1074 icv_off -= 4;
1075 }
1076 CVMX_MF_HSH_IV(tmp1, 0);
1077 *data32 = (uint32_t) (tmp1 >> 32);
1078 IOV_CONSUME(iov, data32, data_i, data_l);
1079 *data32 = (uint32_t) tmp1;
1080 IOV_CONSUME(iov, data32, data_i, data_l);
1081 CVMX_MF_HSH_IV(tmp1, 1);
1082 *data32 = (uint32_t) (tmp1 >> 32);
1083
1084 return 0;
1085 }
1086
1087 /****************************************************************************/
1088 /* DES SHA */
1089
1090 int
1091 octo_des_cbc_sha1_encrypt(
1092 struct octo_sess *od,
1093 struct iovec *iov, size_t iovcnt, size_t iovlen,
1094 int auth_off, int auth_len,
1095 int crypt_off, int crypt_len,
1096 int icv_off, uint8_t *ivp)
1097 {
1098 int next = 0;
1099 union {
1100 uint32_t data32[2];
1101 uint64_t data64[1];
1102 } mydata;
1103 uint64_t *data = &mydata.data64[0];
1104 uint32_t *data32;
1105 uint64_t tmp1, tmp2, tmp3;
1106 int data_i, data_l, alen = auth_len;
1107
1108 dprintf("%s()\n", __func__);
1109
1110 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1111 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1112 (crypt_len & 0x7) ||
1113 (auth_len & 0x7) ||
1114 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1115 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1116 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1117 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1118 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1119 return -EINVAL;
1120 }
1121
1122 IOV_INIT(iov, data32, data_i, data_l);
1123
1124 CVMX_PREFETCH0(ivp);
1125 CVMX_PREFETCH0(od->octo_enckey);
1126
1127 /* load 3DES Key */
1128 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1129 if (od->octo_encklen == 24) {
1130 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1131 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1132 } else if (od->octo_encklen == 8) {
1133 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1134 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1135 } else {
1136 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1137 return -EINVAL;
1138 }
1139
1140 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1141
1142 /* Load SHA1 IV */
1143 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1144 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1145 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1146
1147 while (crypt_off > 0 && auth_off > 0) {
1148 IOV_CONSUME(iov, data32, data_i, data_l);
1149 crypt_off -= 4;
1150 auth_off -= 4;
1151 }
1152
1153 while (crypt_len > 0 || auth_len > 0) {
1154 uint32_t *first = data32;
1155 mydata.data32[0] = *first;
1156 IOV_CONSUME(iov, data32, data_i, data_l);
1157 mydata.data32[1] = *data32;
1158 if (crypt_off <= 0) {
1159 if (crypt_len > 0) {
1160 CVMX_MT_3DES_ENC_CBC(*data);
1161 CVMX_MF_3DES_RESULT(*data);
1162 crypt_len -= 8;
1163 }
1164 } else
1165 crypt_off -= 8;
1166 if (auth_off <= 0) {
1167 if (auth_len > 0) {
1168 CVM_LOAD_SHA_UNIT(*data, next);
1169 auth_len -= 8;
1170 }
1171 } else
1172 auth_off -= 8;
1173 *first = mydata.data32[0];
1174 *data32 = mydata.data32[1];
1175 IOV_CONSUME(iov, data32, data_i, data_l);
1176 }
1177
1178 /* finish the hash */
1179 CVMX_PREFETCH0(od->octo_hmouter);
1180 #if 0
1181 if (__predict_false(inplen)) {
1182 uint64_t tmp = 0;
1183 uint8_t *p = (uint8_t *) & tmp;
1184 p[inplen] = 0x80;
1185 do {
1186 inplen--;
1187 p[inplen] = ((uint8_t *) data)[inplen];
1188 } while (inplen);
1189 CVM_LOAD_SHA_UNIT(tmp, next);
1190 } else {
1191 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1192 }
1193 #else
1194 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1195 #endif
1196
1197 /* Finish Inner hash */
1198 while (next != 7) {
1199 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1200 }
1201 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1202
1203 /* Get the inner hash of HMAC */
1204 CVMX_MF_HSH_IV(tmp1, 0);
1205 CVMX_MF_HSH_IV(tmp2, 1);
1206 tmp3 = 0;
1207 CVMX_MF_HSH_IV(tmp3, 2);
1208
1209 /* Initialize hash unit */
1210 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1211 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1212 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1213
1214 CVMX_MT_HSH_DAT(tmp1, 0);
1215 CVMX_MT_HSH_DAT(tmp2, 1);
1216 tmp3 |= 0x0000000080000000;
1217 CVMX_MT_HSH_DAT(tmp3, 2);
1218 CVMX_MT_HSH_DATZ(3);
1219 CVMX_MT_HSH_DATZ(4);
1220 CVMX_MT_HSH_DATZ(5);
1221 CVMX_MT_HSH_DATZ(6);
1222 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1223
1224 /* save the HMAC */
1225 IOV_INIT(iov, data32, data_i, data_l);
1226 while (icv_off > 0) {
1227 IOV_CONSUME(iov, data32, data_i, data_l);
1228 icv_off -= 4;
1229 }
1230 CVMX_MF_HSH_IV(tmp1, 0);
1231 *data32 = (uint32_t) (tmp1 >> 32);
1232 IOV_CONSUME(iov, data32, data_i, data_l);
1233 *data32 = (uint32_t) tmp1;
1234 IOV_CONSUME(iov, data32, data_i, data_l);
1235 CVMX_MF_HSH_IV(tmp1, 1);
1236 *data32 = (uint32_t) (tmp1 >> 32);
1237
1238 return 0;
1239 }
1240
1241 int
1242 octo_des_cbc_sha1_decrypt(
1243 struct octo_sess *od,
1244 struct iovec *iov, size_t iovcnt, size_t iovlen,
1245 int auth_off, int auth_len,
1246 int crypt_off, int crypt_len,
1247 int icv_off, uint8_t *ivp)
1248 {
1249 int next = 0;
1250 union {
1251 uint32_t data32[2];
1252 uint64_t data64[1];
1253 } mydata;
1254 uint64_t *data = &mydata.data64[0];
1255 uint32_t *data32;
1256 uint64_t tmp1, tmp2, tmp3;
1257 int data_i, data_l, alen = auth_len;
1258
1259 dprintf("%s()\n", __func__);
1260
1261 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1262 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1263 (crypt_len & 0x7) ||
1264 (auth_len & 0x7) ||
1265 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1266 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1267 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1268 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1269 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1270 return -EINVAL;
1271 }
1272
1273 IOV_INIT(iov, data32, data_i, data_l);
1274
1275 CVMX_PREFETCH0(ivp);
1276 CVMX_PREFETCH0(od->octo_enckey);
1277
1278 /* load 3DES Key */
1279 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1280 if (od->octo_encklen == 24) {
1281 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1282 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1283 } else if (od->octo_encklen == 8) {
1284 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1285 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1286 } else {
1287 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1288 return -EINVAL;
1289 }
1290
1291 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1292
1293 /* Load SHA1 IV */
1294 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1295 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1296 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1297
1298 while (crypt_off > 0 && auth_off > 0) {
1299 IOV_CONSUME(iov, data32, data_i, data_l);
1300 crypt_off -= 4;
1301 auth_off -= 4;
1302 }
1303
1304 while (crypt_len > 0 || auth_len > 0) {
1305 uint32_t *first = data32;
1306 mydata.data32[0] = *first;
1307 IOV_CONSUME(iov, data32, data_i, data_l);
1308 mydata.data32[1] = *data32;
1309 if (auth_off <= 0) {
1310 if (auth_len > 0) {
1311 CVM_LOAD_SHA_UNIT(*data, next);
1312 auth_len -= 8;
1313 }
1314 } else
1315 auth_off -= 8;
1316 if (crypt_off <= 0) {
1317 if (crypt_len > 0) {
1318 CVMX_MT_3DES_DEC_CBC(*data);
1319 CVMX_MF_3DES_RESULT(*data);
1320 crypt_len -= 8;
1321 }
1322 } else
1323 crypt_off -= 8;
1324 *first = mydata.data32[0];
1325 *data32 = mydata.data32[1];
1326 IOV_CONSUME(iov, data32, data_i, data_l);
1327 }
1328
1329 /* finish the hash */
1330 CVMX_PREFETCH0(od->octo_hmouter);
1331 #if 0
1332 if (__predict_false(inplen)) {
1333 uint64_t tmp = 0;
1334 uint8_t *p = (uint8_t *) & tmp;
1335 p[inplen] = 0x80;
1336 do {
1337 inplen--;
1338 p[inplen] = ((uint8_t *) data)[inplen];
1339 } while (inplen);
1340 CVM_LOAD_SHA_UNIT(tmp, next);
1341 } else {
1342 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1343 }
1344 #else
1345 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1346 #endif
1347
1348 /* Finish Inner hash */
1349 while (next != 7) {
1350 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1351 }
1352 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1353
1354 /* Get the inner hash of HMAC */
1355 CVMX_MF_HSH_IV(tmp1, 0);
1356 CVMX_MF_HSH_IV(tmp2, 1);
1357 tmp3 = 0;
1358 CVMX_MF_HSH_IV(tmp3, 2);
1359
1360 /* Initialize hash unit */
1361 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1362 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1363 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1364
1365 CVMX_MT_HSH_DAT(tmp1, 0);
1366 CVMX_MT_HSH_DAT(tmp2, 1);
1367 tmp3 |= 0x0000000080000000;
1368 CVMX_MT_HSH_DAT(tmp3, 2);
1369 CVMX_MT_HSH_DATZ(3);
1370 CVMX_MT_HSH_DATZ(4);
1371 CVMX_MT_HSH_DATZ(5);
1372 CVMX_MT_HSH_DATZ(6);
1373 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1374 /* save the HMAC */
1375 IOV_INIT(iov, data32, data_i, data_l);
1376 while (icv_off > 0) {
1377 IOV_CONSUME(iov, data32, data_i, data_l);
1378 icv_off -= 4;
1379 }
1380 CVMX_MF_HSH_IV(tmp1, 0);
1381 *data32 = (uint32_t) (tmp1 >> 32);
1382 IOV_CONSUME(iov, data32, data_i, data_l);
1383 *data32 = (uint32_t) tmp1;
1384 IOV_CONSUME(iov, data32, data_i, data_l);
1385 CVMX_MF_HSH_IV(tmp1, 1);
1386 *data32 = (uint32_t) (tmp1 >> 32);
1387
1388 return 0;
1389 }
1390
1391 /****************************************************************************/
1392 /* AES MD5 */
1393
1394 int
1395 octo_aes_cbc_md5_encrypt(
1396 struct octo_sess *od,
1397 struct iovec *iov, size_t iovcnt, size_t iovlen,
1398 int auth_off, int auth_len,
1399 int crypt_off, int crypt_len,
1400 int icv_off, uint8_t *ivp)
1401 {
1402 int next = 0;
1403 union {
1404 uint32_t data32[2];
1405 uint64_t data64[1];
1406 } mydata[2];
1407 uint64_t *pdata = &mydata[0].data64[0];
1408 uint64_t *data = &mydata[1].data64[0];
1409 uint32_t *data32;
1410 uint64_t tmp1, tmp2;
1411 int data_i, data_l, alen = auth_len;
1412
1413 dprintf("%s()\n", __func__);
1414
1415 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1416 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1417 (crypt_len & 0x7) ||
1418 (auth_len & 0x7) ||
1419 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1420 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1421 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1422 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1423 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1424 return -EINVAL;
1425 }
1426
1427 IOV_INIT(iov, data32, data_i, data_l);
1428
1429 CVMX_PREFETCH0(ivp);
1430 CVMX_PREFETCH0(od->octo_enckey);
1431
1432 /* load AES Key */
1433 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1434 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1435
1436 if (od->octo_encklen == 16) {
1437 CVMX_MT_AES_KEY(0x0, 2);
1438 CVMX_MT_AES_KEY(0x0, 3);
1439 } else if (od->octo_encklen == 24) {
1440 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1441 CVMX_MT_AES_KEY(0x0, 3);
1442 } else if (od->octo_encklen == 32) {
1443 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1444 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1445 } else {
1446 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1447 return -EINVAL;
1448 }
1449 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1450
1451 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1452 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1453
1454 /* Load MD5 IV */
1455 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1456 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1457
1458 while (crypt_off > 0 && auth_off > 0) {
1459 IOV_CONSUME(iov, data32, data_i, data_l);
1460 crypt_off -= 4;
1461 auth_off -= 4;
1462 }
1463
1464 while (crypt_len > 0 || auth_len > 0) {
1465 uint32_t *pdata32[3];
1466
1467 pdata32[0] = data32;
1468 mydata[0].data32[0] = *data32;
1469 IOV_CONSUME(iov, data32, data_i, data_l);
1470
1471 pdata32[1] = data32;
1472 mydata[0].data32[1] = *data32;
1473 IOV_CONSUME(iov, data32, data_i, data_l);
1474
1475 pdata32[2] = data32;
1476 mydata[1].data32[0] = *data32;
1477 IOV_CONSUME(iov, data32, data_i, data_l);
1478
1479 mydata[1].data32[1] = *data32;
1480
1481
1482 if (crypt_off <= 0) {
1483 if (crypt_len > 0) {
1484 CVMX_MT_AES_ENC_CBC0(*pdata);
1485 CVMX_MT_AES_ENC_CBC1(*data);
1486 CVMX_MF_AES_RESULT(*pdata, 0);
1487 CVMX_MF_AES_RESULT(*data, 1);
1488 crypt_len -= 16;
1489 }
1490 } else
1491 crypt_off -= 16;
1492
1493 if (auth_off <= 0) {
1494 if (auth_len > 0) {
1495 CVM_LOAD_MD5_UNIT(*pdata, next);
1496 CVM_LOAD_MD5_UNIT(*data, next);
1497 auth_len -= 16;
1498 }
1499 } else
1500 auth_off -= 16;
1501
1502 *pdata32[0] = mydata[0].data32[0];
1503 *pdata32[1] = mydata[0].data32[1];
1504 *pdata32[2] = mydata[1].data32[0];
1505 *data32 = mydata[1].data32[1];
1506
1507 IOV_CONSUME(iov, data32, data_i, data_l);
1508 }
1509
1510 /* finish the hash */
1511 CVMX_PREFETCH0(od->octo_hmouter);
1512 #if 0
1513 if (__predict_false(inplen)) {
1514 uint64_t tmp = 0;
1515 uint8_t *p = (uint8_t *) & tmp;
1516 p[inplen] = 0x80;
1517 do {
1518 inplen--;
1519 p[inplen] = ((uint8_t *) data)[inplen];
1520 } while (inplen);
1521 CVM_LOAD_MD5_UNIT(tmp, next);
1522 } else {
1523 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1524 }
1525 #else
1526 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1527 #endif
1528
1529 /* Finish Inner hash */
1530 while (next != 7) {
1531 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1532 }
1533 CVMX_ES64(tmp1, ((alen + 64) << 3));
1534 CVM_LOAD_MD5_UNIT(tmp1, next);
1535
1536 /* Get the inner hash of HMAC */
1537 CVMX_MF_HSH_IV(tmp1, 0);
1538 CVMX_MF_HSH_IV(tmp2, 1);
1539
1540 /* Initialize hash unit */
1541 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1542 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1543
1544 CVMX_MT_HSH_DAT(tmp1, 0);
1545 CVMX_MT_HSH_DAT(tmp2, 1);
1546 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1547 CVMX_MT_HSH_DATZ(3);
1548 CVMX_MT_HSH_DATZ(4);
1549 CVMX_MT_HSH_DATZ(5);
1550 CVMX_MT_HSH_DATZ(6);
1551 CVMX_ES64(tmp1, ((64 + 16) << 3));
1552 CVMX_MT_HSH_STARTMD5(tmp1);
1553
1554 /* save the HMAC */
1555 IOV_INIT(iov, data32, data_i, data_l);
1556 while (icv_off > 0) {
1557 IOV_CONSUME(iov, data32, data_i, data_l);
1558 icv_off -= 4;
1559 }
1560 CVMX_MF_HSH_IV(tmp1, 0);
1561 *data32 = (uint32_t) (tmp1 >> 32);
1562 IOV_CONSUME(iov, data32, data_i, data_l);
1563 *data32 = (uint32_t) tmp1;
1564 IOV_CONSUME(iov, data32, data_i, data_l);
1565 CVMX_MF_HSH_IV(tmp1, 1);
1566 *data32 = (uint32_t) (tmp1 >> 32);
1567
1568 return 0;
1569 }
1570
1571 int
1572 octo_aes_cbc_md5_decrypt(
1573 struct octo_sess *od,
1574 struct iovec *iov, size_t iovcnt, size_t iovlen,
1575 int auth_off, int auth_len,
1576 int crypt_off, int crypt_len,
1577 int icv_off, uint8_t *ivp)
1578 {
1579 int next = 0;
1580 union {
1581 uint32_t data32[2];
1582 uint64_t data64[1];
1583 } mydata[2];
1584 uint64_t *pdata = &mydata[0].data64[0];
1585 uint64_t *data = &mydata[1].data64[0];
1586 uint32_t *data32;
1587 uint64_t tmp1, tmp2;
1588 int data_i, data_l, alen = auth_len;
1589
1590 dprintf("%s()\n", __func__);
1591
1592 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1593 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1594 (crypt_len & 0x7) ||
1595 (auth_len & 0x7) ||
1596 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1597 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1598 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1599 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1600 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1601 return -EINVAL;
1602 }
1603
1604 IOV_INIT(iov, data32, data_i, data_l);
1605
1606 CVMX_PREFETCH0(ivp);
1607 CVMX_PREFETCH0(od->octo_enckey);
1608
1609 /* load AES Key */
1610 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1611 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1612
1613 if (od->octo_encklen == 16) {
1614 CVMX_MT_AES_KEY(0x0, 2);
1615 CVMX_MT_AES_KEY(0x0, 3);
1616 } else if (od->octo_encklen == 24) {
1617 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1618 CVMX_MT_AES_KEY(0x0, 3);
1619 } else if (od->octo_encklen == 32) {
1620 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1621 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1622 } else {
1623 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1624 return -EINVAL;
1625 }
1626 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1627
1628 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1629 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1630
1631 /* Load MD5 IV */
1632 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1633 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1634
1635 while (crypt_off > 0 && auth_off > 0) {
1636 IOV_CONSUME(iov, data32, data_i, data_l);
1637 crypt_off -= 4;
1638 auth_off -= 4;
1639 }
1640
1641 while (crypt_len > 0 || auth_len > 0) {
1642 uint32_t *pdata32[3];
1643
1644 pdata32[0] = data32;
1645 mydata[0].data32[0] = *data32;
1646 IOV_CONSUME(iov, data32, data_i, data_l);
1647 pdata32[1] = data32;
1648 mydata[0].data32[1] = *data32;
1649 IOV_CONSUME(iov, data32, data_i, data_l);
1650 pdata32[2] = data32;
1651 mydata[1].data32[0] = *data32;
1652 IOV_CONSUME(iov, data32, data_i, data_l);
1653 mydata[1].data32[1] = *data32;
1654
1655 if (auth_off <= 0) {
1656 if (auth_len > 0) {
1657 CVM_LOAD_MD5_UNIT(*pdata, next);
1658 CVM_LOAD_MD5_UNIT(*data, next);
1659 auth_len -= 16;
1660 }
1661 } else
1662 auth_off -= 16;
1663
1664 if (crypt_off <= 0) {
1665 if (crypt_len > 0) {
1666 CVMX_MT_AES_DEC_CBC0(*pdata);
1667 CVMX_MT_AES_DEC_CBC1(*data);
1668 CVMX_MF_AES_RESULT(*pdata, 0);
1669 CVMX_MF_AES_RESULT(*data, 1);
1670 crypt_len -= 16;
1671 }
1672 } else
1673 crypt_off -= 16;
1674
1675 *pdata32[0] = mydata[0].data32[0];
1676 *pdata32[1] = mydata[0].data32[1];
1677 *pdata32[2] = mydata[1].data32[0];
1678 *data32 = mydata[1].data32[1];
1679
1680 IOV_CONSUME(iov, data32, data_i, data_l);
1681 }
1682
1683 /* finish the hash */
1684 CVMX_PREFETCH0(od->octo_hmouter);
1685 #if 0
1686 if (__predict_false(inplen)) {
1687 uint64_t tmp = 0;
1688 uint8_t *p = (uint8_t *) & tmp;
1689 p[inplen] = 0x80;
1690 do {
1691 inplen--;
1692 p[inplen] = ((uint8_t *) data)[inplen];
1693 } while (inplen);
1694 CVM_LOAD_MD5_UNIT(tmp, next);
1695 } else {
1696 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1697 }
1698 #else
1699 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1700 #endif
1701
1702 /* Finish Inner hash */
1703 while (next != 7) {
1704 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1705 }
1706 CVMX_ES64(tmp1, ((alen + 64) << 3));
1707 CVM_LOAD_MD5_UNIT(tmp1, next);
1708
1709 /* Get the inner hash of HMAC */
1710 CVMX_MF_HSH_IV(tmp1, 0);
1711 CVMX_MF_HSH_IV(tmp2, 1);
1712
1713 /* Initialize hash unit */
1714 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1715 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1716
1717 CVMX_MT_HSH_DAT(tmp1, 0);
1718 CVMX_MT_HSH_DAT(tmp2, 1);
1719 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1720 CVMX_MT_HSH_DATZ(3);
1721 CVMX_MT_HSH_DATZ(4);
1722 CVMX_MT_HSH_DATZ(5);
1723 CVMX_MT_HSH_DATZ(6);
1724 CVMX_ES64(tmp1, ((64 + 16) << 3));
1725 CVMX_MT_HSH_STARTMD5(tmp1);
1726
1727 /* save the HMAC */
1728 IOV_INIT(iov, data32, data_i, data_l);
1729 while (icv_off > 0) {
1730 IOV_CONSUME(iov, data32, data_i, data_l);
1731 icv_off -= 4;
1732 }
1733 CVMX_MF_HSH_IV(tmp1, 0);
1734 *data32 = (uint32_t) (tmp1 >> 32);
1735 IOV_CONSUME(iov, data32, data_i, data_l);
1736 *data32 = (uint32_t) tmp1;
1737 IOV_CONSUME(iov, data32, data_i, data_l);
1738 CVMX_MF_HSH_IV(tmp1, 1);
1739 *data32 = (uint32_t) (tmp1 >> 32);
1740
1741 return 0;
1742 }
1743
1744 /****************************************************************************/
1745 /* AES SHA1 */
1746
1747 int
1748 octo_aes_cbc_sha1_encrypt(
1749 struct octo_sess *od,
1750 struct iovec *iov, size_t iovcnt, size_t iovlen,
1751 int auth_off, int auth_len,
1752 int crypt_off, int crypt_len,
1753 int icv_off, uint8_t *ivp)
1754 {
1755 int next = 0;
1756 union {
1757 uint32_t data32[2];
1758 uint64_t data64[1];
1759 } mydata[2];
1760 uint64_t *pdata = &mydata[0].data64[0];
1761 uint64_t *data = &mydata[1].data64[0];
1762 uint32_t *data32;
1763 uint64_t tmp1, tmp2, tmp3;
1764 int data_i, data_l, alen = auth_len;
1765
1766 dprintf("%s()\n", __func__);
1767
1768 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1769 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1770 (crypt_len & 0x7) ||
1771 (auth_len & 0x7) ||
1772 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1773 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1774 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1775 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1776 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1777 return -EINVAL;
1778 }
1779
1780 IOV_INIT(iov, data32, data_i, data_l);
1781
1782 CVMX_PREFETCH0(ivp);
1783 CVMX_PREFETCH0(od->octo_enckey);
1784
1785 /* load AES Key */
1786 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1787 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1788
1789 if (od->octo_encklen == 16) {
1790 CVMX_MT_AES_KEY(0x0, 2);
1791 CVMX_MT_AES_KEY(0x0, 3);
1792 } else if (od->octo_encklen == 24) {
1793 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1794 CVMX_MT_AES_KEY(0x0, 3);
1795 } else if (od->octo_encklen == 32) {
1796 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1797 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1798 } else {
1799 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1800 return -EINVAL;
1801 }
1802 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1803
1804 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1805 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1806
1807 /* Load SHA IV */
1808 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1809 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1810 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1811
1812 while (crypt_off > 0 && auth_off > 0) {
1813 IOV_CONSUME(iov, data32, data_i, data_l);
1814 crypt_off -= 4;
1815 auth_off -= 4;
1816 }
1817
1818 while (crypt_len > 0 || auth_len > 0) {
1819 uint32_t *pdata32[3];
1820
1821 pdata32[0] = data32;
1822 mydata[0].data32[0] = *data32;
1823 IOV_CONSUME(iov, data32, data_i, data_l);
1824 pdata32[1] = data32;
1825 mydata[0].data32[1] = *data32;
1826 IOV_CONSUME(iov, data32, data_i, data_l);
1827 pdata32[2] = data32;
1828 mydata[1].data32[0] = *data32;
1829 IOV_CONSUME(iov, data32, data_i, data_l);
1830 mydata[1].data32[1] = *data32;
1831
1832
1833 if (crypt_off <= 0) {
1834 if (crypt_len > 0) {
1835 CVMX_MT_AES_ENC_CBC0(*pdata);
1836 CVMX_MT_AES_ENC_CBC1(*data);
1837 CVMX_MF_AES_RESULT(*pdata, 0);
1838 CVMX_MF_AES_RESULT(*data, 1);
1839 crypt_len -= 16;
1840 }
1841 } else
1842 crypt_off -= 16;
1843
1844 if (auth_off <= 0) {
1845 if (auth_len > 0) {
1846 CVM_LOAD_SHA_UNIT(*pdata, next);
1847 CVM_LOAD_SHA_UNIT(*data, next);
1848 auth_len -= 16;
1849 }
1850 } else
1851 auth_off -= 16;
1852
1853 *pdata32[0] = mydata[0].data32[0];
1854 *pdata32[1] = mydata[0].data32[1];
1855 *pdata32[2] = mydata[1].data32[0];
1856 *data32 = mydata[1].data32[1];
1857
1858 IOV_CONSUME(iov, data32, data_i, data_l);
1859 }
1860
1861 /* finish the hash */
1862 CVMX_PREFETCH0(od->octo_hmouter);
1863 #if 0
1864 if (__predict_false(inplen)) {
1865 uint64_t tmp = 0;
1866 uint8_t *p = (uint8_t *) & tmp;
1867 p[inplen] = 0x80;
1868 do {
1869 inplen--;
1870 p[inplen] = ((uint8_t *) data)[inplen];
1871 } while (inplen);
1872 CVM_LOAD_SHA_UNIT(tmp, next);
1873 } else {
1874 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1875 }
1876 #else
1877 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1878 #endif
1879
1880 /* Finish Inner hash */
1881 while (next != 7) {
1882 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1883 }
1884 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1885
1886 /* Get the inner hash of HMAC */
1887 CVMX_MF_HSH_IV(tmp1, 0);
1888 CVMX_MF_HSH_IV(tmp2, 1);
1889 tmp3 = 0;
1890 CVMX_MF_HSH_IV(tmp3, 2);
1891
1892 /* Initialize hash unit */
1893 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1894 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1895 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1896
1897 CVMX_MT_HSH_DAT(tmp1, 0);
1898 CVMX_MT_HSH_DAT(tmp2, 1);
1899 tmp3 |= 0x0000000080000000;
1900 CVMX_MT_HSH_DAT(tmp3, 2);
1901 CVMX_MT_HSH_DATZ(3);
1902 CVMX_MT_HSH_DATZ(4);
1903 CVMX_MT_HSH_DATZ(5);
1904 CVMX_MT_HSH_DATZ(6);
1905 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1906
1907 /* finish the hash */
1908 CVMX_PREFETCH0(od->octo_hmouter);
1909 #if 0
1910 if (__predict_false(inplen)) {
1911 uint64_t tmp = 0;
1912 uint8_t *p = (uint8_t *) & tmp;
1913 p[inplen] = 0x80;
1914 do {
1915 inplen--;
1916 p[inplen] = ((uint8_t *) data)[inplen];
1917 } while (inplen);
1918 CVM_LOAD_MD5_UNIT(tmp, next);
1919 } else {
1920 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1921 }
1922 #else
1923 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1924 #endif
1925
1926 /* save the HMAC */
1927 IOV_INIT(iov, data32, data_i, data_l);
1928 while (icv_off > 0) {
1929 IOV_CONSUME(iov, data32, data_i, data_l);
1930 icv_off -= 4;
1931 }
1932 CVMX_MF_HSH_IV(tmp1, 0);
1933 *data32 = (uint32_t) (tmp1 >> 32);
1934 IOV_CONSUME(iov, data32, data_i, data_l);
1935 *data32 = (uint32_t) tmp1;
1936 IOV_CONSUME(iov, data32, data_i, data_l);
1937 CVMX_MF_HSH_IV(tmp1, 1);
1938 *data32 = (uint32_t) (tmp1 >> 32);
1939
1940 return 0;
1941 }
1942
1943 int
1944 octo_aes_cbc_sha1_decrypt(
1945 struct octo_sess *od,
1946 struct iovec *iov, size_t iovcnt, size_t iovlen,
1947 int auth_off, int auth_len,
1948 int crypt_off, int crypt_len,
1949 int icv_off, uint8_t *ivp)
1950 {
1951 int next = 0;
1952 union {
1953 uint32_t data32[2];
1954 uint64_t data64[1];
1955 } mydata[2];
1956 uint64_t *pdata = &mydata[0].data64[0];
1957 uint64_t *data = &mydata[1].data64[0];
1958 uint32_t *data32;
1959 uint64_t tmp1, tmp2, tmp3;
1960 int data_i, data_l, alen = auth_len;
1961
1962 dprintf("%s()\n", __func__);
1963
1964 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1965 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1966 (crypt_len & 0x7) ||
1967 (auth_len & 0x7) ||
1968 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1969 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1970 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1971 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1972 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1973 return -EINVAL;
1974 }
1975
1976 IOV_INIT(iov, data32, data_i, data_l);
1977
1978 CVMX_PREFETCH0(ivp);
1979 CVMX_PREFETCH0(od->octo_enckey);
1980
1981 /* load AES Key */
1982 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1983 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1984
1985 if (od->octo_encklen == 16) {
1986 CVMX_MT_AES_KEY(0x0, 2);
1987 CVMX_MT_AES_KEY(0x0, 3);
1988 } else if (od->octo_encklen == 24) {
1989 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1990 CVMX_MT_AES_KEY(0x0, 3);
1991 } else if (od->octo_encklen == 32) {
1992 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1993 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1994 } else {
1995 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1996 return -EINVAL;
1997 }
1998 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1999
2000 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
2001 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
2002
2003 /* Load MD5 IV */
2004 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
2005 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
2006 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
2007
2008 while (crypt_off > 0 && auth_off > 0) {
2009 IOV_CONSUME(iov, data32, data_i, data_l);
2010 crypt_off -= 4;
2011 auth_off -= 4;
2012 }
2013
2014 while (crypt_len > 0 || auth_len > 0) {
2015 uint32_t *pdata32[3];
2016
2017 pdata32[0] = data32;
2018 mydata[0].data32[0] = *data32;
2019 IOV_CONSUME(iov, data32, data_i, data_l);
2020 pdata32[1] = data32;
2021 mydata[0].data32[1] = *data32;
2022 IOV_CONSUME(iov, data32, data_i, data_l);
2023 pdata32[2] = data32;
2024 mydata[1].data32[0] = *data32;
2025 IOV_CONSUME(iov, data32, data_i, data_l);
2026 mydata[1].data32[1] = *data32;
2027
2028 if (auth_off <= 0) {
2029 if (auth_len > 0) {
2030 CVM_LOAD_SHA_UNIT(*pdata, next);
2031 CVM_LOAD_SHA_UNIT(*data, next);
2032 auth_len -= 16;
2033 }
2034 } else
2035 auth_off -= 16;
2036
2037 if (crypt_off <= 0) {
2038 if (crypt_len > 0) {
2039 CVMX_MT_AES_DEC_CBC0(*pdata);
2040 CVMX_MT_AES_DEC_CBC1(*data);
2041 CVMX_MF_AES_RESULT(*pdata, 0);
2042 CVMX_MF_AES_RESULT(*data, 1);
2043 crypt_len -= 16;
2044 }
2045 } else
2046 crypt_off -= 16;
2047
2048 *pdata32[0] = mydata[0].data32[0];
2049 *pdata32[1] = mydata[0].data32[1];
2050 *pdata32[2] = mydata[1].data32[0];
2051 *data32 = mydata[1].data32[1];
2052
2053 IOV_CONSUME(iov, data32, data_i, data_l);
2054 }
2055
2056 /* finish the hash */
2057 CVMX_PREFETCH0(od->octo_hmouter);
2058 #if 0
2059 if (__predict_false(inplen)) {
2060 uint64_t tmp = 0;
2061 uint8_t *p = (uint8_t *) & tmp;
2062 p[inplen] = 0x80;
2063 do {
2064 inplen--;
2065 p[inplen] = ((uint8_t *) data)[inplen];
2066 } while (inplen);
2067 CVM_LOAD_SHA_UNIT(tmp, next);
2068 } else {
2069 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2070 }
2071 #else
2072 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2073 #endif
2074
2075 /* Finish Inner hash */
2076 while (next != 7) {
2077 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
2078 }
2079 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
2080
2081 /* Get the inner hash of HMAC */
2082 CVMX_MF_HSH_IV(tmp1, 0);
2083 CVMX_MF_HSH_IV(tmp2, 1);
2084 tmp3 = 0;
2085 CVMX_MF_HSH_IV(tmp3, 2);
2086
2087 /* Initialize hash unit */
2088 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
2089 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
2090 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
2091
2092 CVMX_MT_HSH_DAT(tmp1, 0);
2093 CVMX_MT_HSH_DAT(tmp2, 1);
2094 tmp3 |= 0x0000000080000000;
2095 CVMX_MT_HSH_DAT(tmp3, 2);
2096 CVMX_MT_HSH_DATZ(3);
2097 CVMX_MT_HSH_DATZ(4);
2098 CVMX_MT_HSH_DATZ(5);
2099 CVMX_MT_HSH_DATZ(6);
2100 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
2101
2102 /* finish the hash */
2103 CVMX_PREFETCH0(od->octo_hmouter);
2104 #if 0
2105 if (__predict_false(inplen)) {
2106 uint64_t tmp = 0;
2107 uint8_t *p = (uint8_t *) & tmp;
2108 p[inplen] = 0x80;
2109 do {
2110 inplen--;
2111 p[inplen] = ((uint8_t *) data)[inplen];
2112 } while (inplen);
2113 CVM_LOAD_MD5_UNIT(tmp, next);
2114 } else {
2115 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2116 }
2117 #else
2118 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2119 #endif
2120
2121 /* save the HMAC */
2122 IOV_INIT(iov, data32, data_i, data_l);
2123 while (icv_off > 0) {
2124 IOV_CONSUME(iov, data32, data_i, data_l);
2125 icv_off -= 4;
2126 }
2127 CVMX_MF_HSH_IV(tmp1, 0);
2128 *data32 = (uint32_t) (tmp1 >> 32);
2129 IOV_CONSUME(iov, data32, data_i, data_l);
2130 *data32 = (uint32_t) tmp1;
2131 IOV_CONSUME(iov, data32, data_i, data_l);
2132 CVMX_MF_HSH_IV(tmp1, 1);
2133 *data32 = (uint32_t) (tmp1 >> 32);
2134
2135 return 0;
2136 }
2137
2138 /****************************************************************************/
Cache object: 9b8dba38541b0a3799c108b1f007e0e9
|