Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/mips/cavium/cryptocteon/cavium_crypto.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * vim:sw=4 ts=8 3 */ 4 /*- 5 * SPDX-License-Identifier: BSD-4-Clause 6 * 7 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com> 8 * 9 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 10 * reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions are met: 14 * 1. Redistributions of source code must retain the above copyright notice, 15 * this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright notice, 17 * this list of conditions and the following disclaimer in the documentation 18 * and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by Cavium Networks 22 * 4. Cavium Networks' name may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * This Software, including technical data, may be subject to U.S. export 26 * control laws, including the U.S. Export Administration Act and its 27 * associated regulations, and may be subject to export or import regulations 28 * in other countries. You warrant that You will comply strictly in all 29 * respects with all such regulations and acknowledge that you have the 30 * responsibility to obtain licenses to export, re-export or import the 31 * Software. 32 * 33 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND 34 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, 35 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE 36 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR 37 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM 38 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, 39 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF 40 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR 41 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR 42 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 43 */ 44 /****************************************************************************/ 45 46 #include <sys/cdefs.h> 47 __FBSDID("$FreeBSD$"); 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/kernel.h> 52 #include <sys/module.h> 53 #include <sys/malloc.h> 54 #include <sys/uio.h> 55 56 #include <opencrypto/cryptodev.h> 57 58 #include <contrib/octeon-sdk/cvmx.h> 59 60 #include <mips/cavium/cryptocteon/cryptocteonvar.h> 61 62 /****************************************************************************/ 63 64 #define IOV_INIT(iov, ptr, idx, len) \ 65 do { \ 66 (idx) = 0; \ 67 (ptr) = (iov)[(idx)].iov_base; \ 68 (len) = (iov)[(idx)].iov_len; \ 69 } while (0) 70 71 /* 72 * XXX 73 * It would be better if this were an IOV_READ/IOV_WRITE macro instead so 74 * that we could detect overflow before it happens rather than right after, 75 * which is especially bad since there is usually no IOV_CONSUME after the 76 * final read or write. 77 */ 78 #define IOV_CONSUME(iov, ptr, idx, len) \ 79 do { \ 80 if ((len) > sizeof *(ptr)) { \ 81 (len) -= sizeof *(ptr); \ 82 (ptr)++; \ 83 } else { \ 84 if ((len) != sizeof *(ptr)) \ 85 panic("%s: went past end of iovec.", __func__); \ 86 (idx)++; \ 87 (ptr) = (iov)[(idx)].iov_base; \ 88 (len) = (iov)[(idx)].iov_len; \ 89 } \ 90 } while (0) 91 92 #define ESP_HEADER_LENGTH 8 93 #define DES_CBC_IV_LENGTH 8 94 #define AES_CBC_IV_LENGTH 16 95 #define ESP_HMAC_LEN 12 96 97 #define ESP_HEADER_LENGTH 8 98 #define DES_CBC_IV_LENGTH 8 99 100 /****************************************************************************/ 101 102 #define CVM_LOAD_SHA_UNIT(dat, next) { \ 103 if (next == 0) { \ 104 next = 1; \ 105 CVMX_MT_HSH_DAT (dat, 0); \ 106 } else if (next == 1) { \ 107 next = 2; \ 108 CVMX_MT_HSH_DAT (dat, 1); \ 109 } else if (next == 2) { \ 110 next = 3; \ 111 CVMX_MT_HSH_DAT (dat, 2); \ 112 } else if (next == 3) { \ 113 next = 4; \ 114 CVMX_MT_HSH_DAT (dat, 3); \ 115 } else if (next == 4) { \ 116 next = 5; \ 117 CVMX_MT_HSH_DAT (dat, 4); \ 118 } else if (next == 5) { \ 119 next = 6; \ 120 CVMX_MT_HSH_DAT (dat, 5); \ 121 } else if (next == 6) { \ 122 next = 7; \ 123 CVMX_MT_HSH_DAT (dat, 6); \ 124 } else { \ 125 CVMX_MT_HSH_STARTSHA (dat); \ 126 next = 0; \ 127 } \ 128 } 129 130 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ 131 if (next == 0) { \ 132 CVMX_MT_HSH_DAT (dat1, 0); \ 133 CVMX_MT_HSH_DAT (dat2, 1); \ 134 next = 2; \ 135 } else if (next == 1) { \ 136 CVMX_MT_HSH_DAT (dat1, 1); \ 137 CVMX_MT_HSH_DAT (dat2, 2); \ 138 next = 3; \ 139 } else if (next == 2) { \ 140 CVMX_MT_HSH_DAT (dat1, 2); \ 141 CVMX_MT_HSH_DAT (dat2, 3); \ 142 next = 4; \ 143 } else if (next == 3) { \ 144 CVMX_MT_HSH_DAT (dat1, 3); \ 145 CVMX_MT_HSH_DAT (dat2, 4); \ 146 next = 5; \ 147 } else if (next == 4) { \ 148 CVMX_MT_HSH_DAT (dat1, 4); \ 149 CVMX_MT_HSH_DAT (dat2, 5); \ 150 next = 6; \ 151 } else if (next == 5) { \ 152 CVMX_MT_HSH_DAT (dat1, 5); \ 153 CVMX_MT_HSH_DAT (dat2, 6); \ 154 next = 7; \ 155 } else if (next == 6) { \ 156 CVMX_MT_HSH_DAT (dat1, 6); \ 157 CVMX_MT_HSH_STARTSHA (dat2); \ 158 next = 0; \ 159 } else { \ 160 CVMX_MT_HSH_STARTSHA (dat1); \ 161 CVMX_MT_HSH_DAT (dat2, 0); \ 162 next = 1; \ 163 } \ 164 } 165 166 /****************************************************************************/ 167 168 #define CVM_LOAD_MD5_UNIT(dat, next) { \ 169 if (next == 0) { \ 170 next = 1; \ 171 CVMX_MT_HSH_DAT (dat, 0); \ 172 } else if (next == 1) { \ 173 next = 2; \ 174 CVMX_MT_HSH_DAT (dat, 1); \ 175 } else if (next == 2) { \ 176 next = 3; \ 177 CVMX_MT_HSH_DAT (dat, 2); \ 178 } else if (next == 3) { \ 179 next = 4; \ 180 CVMX_MT_HSH_DAT (dat, 3); \ 181 } else if (next == 4) { \ 182 next = 5; \ 183 CVMX_MT_HSH_DAT (dat, 4); \ 184 } else if (next == 5) { \ 185 next = 6; \ 186 CVMX_MT_HSH_DAT (dat, 5); \ 187 } else if (next == 6) { \ 188 next = 7; \ 189 CVMX_MT_HSH_DAT (dat, 6); \ 190 } else { \ 191 CVMX_MT_HSH_STARTMD5 (dat); \ 192 next = 0; \ 193 } \ 194 } 195 196 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ 197 if (next == 0) { \ 198 CVMX_MT_HSH_DAT (dat1, 0); \ 199 CVMX_MT_HSH_DAT (dat2, 1); \ 200 next = 2; \ 201 } else if (next == 1) { \ 202 CVMX_MT_HSH_DAT (dat1, 1); \ 203 CVMX_MT_HSH_DAT (dat2, 2); \ 204 next = 3; \ 205 } else if (next == 2) { \ 206 CVMX_MT_HSH_DAT (dat1, 2); \ 207 CVMX_MT_HSH_DAT (dat2, 3); \ 208 next = 4; \ 209 } else if (next == 3) { \ 210 CVMX_MT_HSH_DAT (dat1, 3); \ 211 CVMX_MT_HSH_DAT (dat2, 4); \ 212 next = 5; \ 213 } else if (next == 4) { \ 214 CVMX_MT_HSH_DAT (dat1, 4); \ 215 CVMX_MT_HSH_DAT (dat2, 5); \ 216 next = 6; \ 217 } else if (next == 5) { \ 218 CVMX_MT_HSH_DAT (dat1, 5); \ 219 CVMX_MT_HSH_DAT (dat2, 6); \ 220 next = 7; \ 221 } else if (next == 6) { \ 222 CVMX_MT_HSH_DAT (dat1, 6); \ 223 CVMX_MT_HSH_STARTMD5 (dat2); \ 224 next = 0; \ 225 } else { \ 226 CVMX_MT_HSH_STARTMD5 (dat1); \ 227 CVMX_MT_HSH_DAT (dat2, 0); \ 228 next = 1; \ 229 } \ 230 } 231 232 /****************************************************************************/ 233 234 void 235 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer) 236 { 237 uint8_t hash_key[64]; 238 uint64_t *key1; 239 register uint64_t xor1 = 0x3636363636363636ULL; 240 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; 241 242 dprintf("%s()\n", __func__); 243 244 memset(hash_key, 0, sizeof(hash_key)); 245 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); 246 key1 = (uint64_t *) hash_key; 247 if (auth) { 248 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); 249 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); 250 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); 251 } else { 252 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); 253 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); 254 } 255 256 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); 257 key1++; 258 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); 259 key1++; 260 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); 261 key1++; 262 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); 263 key1++; 264 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); 265 key1++; 266 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); 267 key1++; 268 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); 269 key1++; 270 if (auth) 271 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); 272 else 273 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); 274 275 CVMX_MF_HSH_IV(inner[0], 0); 276 CVMX_MF_HSH_IV(inner[1], 1); 277 if (auth) { 278 inner[2] = 0; 279 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); 280 } 281 282 memset(hash_key, 0, sizeof(hash_key)); 283 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); 284 key1 = (uint64_t *) hash_key; 285 if (auth) { 286 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); 287 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); 288 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); 289 } else { 290 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); 291 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); 292 } 293 294 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); 295 key1++; 296 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); 297 key1++; 298 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); 299 key1++; 300 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); 301 key1++; 302 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); 303 key1++; 304 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); 305 key1++; 306 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); 307 key1++; 308 if (auth) 309 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); 310 else 311 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); 312 313 CVMX_MF_HSH_IV(outer[0], 0); 314 CVMX_MF_HSH_IV(outer[1], 1); 315 if (auth) { 316 outer[2] = 0; 317 CVMX_MF_HSH_IV(outer[2], 2); 318 } 319 return; 320 } 321 322 /****************************************************************************/ 323 /* DES functions */ 324 325 int 326 octo_des_cbc_encrypt( 327 struct octo_sess *od, 328 struct iovec *iov, size_t iovcnt, size_t iovlen, 329 int auth_off, int auth_len, 330 int crypt_off, int crypt_len, 331 int icv_off, uint8_t *ivp) 332 { 333 uint64_t *data; 334 int data_i, data_l; 335 336 dprintf("%s()\n", __func__); 337 338 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 339 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 340 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 341 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 342 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 343 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 344 return -EINVAL; 345 } 346 347 IOV_INIT(iov, data, data_i, data_l); 348 349 CVMX_PREFETCH0(ivp); 350 CVMX_PREFETCH0(od->octo_enckey); 351 352 353 /* load 3DES Key */ 354 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 355 if (od->octo_encklen == 24) { 356 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 358 } else if (od->octo_encklen == 8) { 359 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 360 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 361 } else { 362 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 363 return -EINVAL; 364 } 365 366 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 367 368 while (crypt_off > 0) { 369 IOV_CONSUME(iov, data, data_i, data_l); 370 crypt_off -= 8; 371 } 372 373 while (crypt_len > 0) { 374 CVMX_MT_3DES_ENC_CBC(*data); 375 CVMX_MF_3DES_RESULT(*data); 376 IOV_CONSUME(iov, data, data_i, data_l); 377 crypt_len -= 8; 378 } 379 380 return 0; 381 } 382 383 384 int 385 octo_des_cbc_decrypt( 386 struct octo_sess *od, 387 struct iovec *iov, size_t iovcnt, size_t iovlen, 388 int auth_off, int auth_len, 389 int crypt_off, int crypt_len, 390 int icv_off, uint8_t *ivp) 391 { 392 uint64_t *data; 393 int data_i, data_l; 394 395 dprintf("%s()\n", __func__); 396 397 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 398 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 399 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 400 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 401 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 402 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 403 return -EINVAL; 404 } 405 406 IOV_INIT(iov, data, data_i, data_l); 407 408 CVMX_PREFETCH0(ivp); 409 CVMX_PREFETCH0(od->octo_enckey); 410 411 /* load 3DES Key */ 412 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 413 if (od->octo_encklen == 24) { 414 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 415 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 416 } else if (od->octo_encklen == 8) { 417 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 418 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 419 } else { 420 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 421 return -EINVAL; 422 } 423 424 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 425 426 while (crypt_off > 0) { 427 IOV_CONSUME(iov, data, data_i, data_l); 428 crypt_off -= 8; 429 } 430 431 while (crypt_len > 0) { 432 CVMX_MT_3DES_DEC_CBC(*data); 433 CVMX_MF_3DES_RESULT(*data); 434 IOV_CONSUME(iov, data, data_i, data_l); 435 crypt_len -= 8; 436 } 437 438 return 0; 439 } 440 441 /****************************************************************************/ 442 /* AES functions */ 443 444 int 445 octo_aes_cbc_encrypt( 446 struct octo_sess *od, 447 struct iovec *iov, size_t iovcnt, size_t iovlen, 448 int auth_off, int auth_len, 449 int crypt_off, int crypt_len, 450 int icv_off, uint8_t *ivp) 451 { 452 uint64_t *data, *pdata; 453 int data_i, data_l; 454 455 dprintf("%s()\n", __func__); 456 457 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 458 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 459 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 460 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 461 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 462 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 463 return -EINVAL; 464 } 465 466 IOV_INIT(iov, data, data_i, data_l); 467 468 CVMX_PREFETCH0(ivp); 469 CVMX_PREFETCH0(od->octo_enckey); 470 471 /* load AES Key */ 472 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 473 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 474 475 if (od->octo_encklen == 16) { 476 CVMX_MT_AES_KEY(0x0, 2); 477 CVMX_MT_AES_KEY(0x0, 3); 478 } else if (od->octo_encklen == 24) { 479 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 480 CVMX_MT_AES_KEY(0x0, 3); 481 } else if (od->octo_encklen == 32) { 482 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 483 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 484 } else { 485 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 486 return -EINVAL; 487 } 488 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 489 490 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 491 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 492 493 while (crypt_off > 0) { 494 IOV_CONSUME(iov, data, data_i, data_l); 495 crypt_off -= 8; 496 } 497 498 while (crypt_len > 0) { 499 pdata = data; 500 CVMX_MT_AES_ENC_CBC0(*data); 501 IOV_CONSUME(iov, data, data_i, data_l); 502 CVMX_MT_AES_ENC_CBC1(*data); 503 CVMX_MF_AES_RESULT(*pdata, 0); 504 CVMX_MF_AES_RESULT(*data, 1); 505 IOV_CONSUME(iov, data, data_i, data_l); 506 crypt_len -= 16; 507 } 508 509 return 0; 510 } 511 512 513 int 514 octo_aes_cbc_decrypt( 515 struct octo_sess *od, 516 struct iovec *iov, size_t iovcnt, size_t iovlen, 517 int auth_off, int auth_len, 518 int crypt_off, int crypt_len, 519 int icv_off, uint8_t *ivp) 520 { 521 uint64_t *data, *pdata; 522 int data_i, data_l; 523 524 dprintf("%s()\n", __func__); 525 526 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 527 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 528 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 529 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 530 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 531 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 532 return -EINVAL; 533 } 534 535 IOV_INIT(iov, data, data_i, data_l); 536 537 CVMX_PREFETCH0(ivp); 538 CVMX_PREFETCH0(od->octo_enckey); 539 540 /* load AES Key */ 541 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 542 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 543 544 if (od->octo_encklen == 16) { 545 CVMX_MT_AES_KEY(0x0, 2); 546 CVMX_MT_AES_KEY(0x0, 3); 547 } else if (od->octo_encklen == 24) { 548 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 549 CVMX_MT_AES_KEY(0x0, 3); 550 } else if (od->octo_encklen == 32) { 551 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 552 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 553 } else { 554 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 555 return -EINVAL; 556 } 557 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 558 559 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 560 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 561 562 while (crypt_off > 0) { 563 IOV_CONSUME(iov, data, data_i, data_l); 564 crypt_off -= 8; 565 } 566 567 while (crypt_len > 0) { 568 pdata = data; 569 CVMX_MT_AES_DEC_CBC0(*data); 570 IOV_CONSUME(iov, data, data_i, data_l); 571 CVMX_MT_AES_DEC_CBC1(*data); 572 CVMX_MF_AES_RESULT(*pdata, 0); 573 CVMX_MF_AES_RESULT(*data, 1); 574 IOV_CONSUME(iov, data, data_i, data_l); 575 crypt_len -= 16; 576 } 577 578 return 0; 579 } 580 581 /****************************************************************************/ 582 /* MD5 */ 583 584 int 585 octo_null_md5_encrypt( 586 struct octo_sess *od, 587 struct iovec *iov, size_t iovcnt, size_t iovlen, 588 int auth_off, int auth_len, 589 int crypt_off, int crypt_len, 590 int icv_off, uint8_t *ivp) 591 { 592 int next = 0; 593 uint64_t *data; 594 uint64_t tmp1, tmp2; 595 int data_i, data_l, alen = auth_len; 596 597 dprintf("%s()\n", __func__); 598 599 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || 600 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { 601 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 602 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 603 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 604 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 605 return -EINVAL; 606 } 607 608 IOV_INIT(iov, data, data_i, data_l); 609 610 /* Load MD5 IV */ 611 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 612 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 613 614 while (auth_off > 0) { 615 IOV_CONSUME(iov, data, data_i, data_l); 616 auth_off -= 8; 617 } 618 619 while (auth_len > 0) { 620 CVM_LOAD_MD5_UNIT(*data, next); 621 auth_len -= 8; 622 IOV_CONSUME(iov, data, data_i, data_l); 623 } 624 625 /* finish the hash */ 626 CVMX_PREFETCH0(od->octo_hmouter); 627 #if 0 628 if (__predict_false(inplen)) { 629 uint64_t tmp = 0; 630 uint8_t *p = (uint8_t *) & tmp; 631 p[inplen] = 0x80; 632 do { 633 inplen--; 634 p[inplen] = ((uint8_t *) data)[inplen]; 635 } while (inplen); 636 CVM_LOAD_MD5_UNIT(tmp, next); 637 } else { 638 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 639 } 640 #else 641 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 642 #endif 643 644 /* Finish Inner hash */ 645 while (next != 7) { 646 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 647 } 648 CVMX_ES64(tmp1, ((alen + 64) << 3)); 649 CVM_LOAD_MD5_UNIT(tmp1, next); 650 651 /* Get the inner hash of HMAC */ 652 CVMX_MF_HSH_IV(tmp1, 0); 653 CVMX_MF_HSH_IV(tmp2, 1); 654 655 /* Initialize hash unit */ 656 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 657 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 658 659 CVMX_MT_HSH_DAT(tmp1, 0); 660 CVMX_MT_HSH_DAT(tmp2, 1); 661 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 662 CVMX_MT_HSH_DATZ(3); 663 CVMX_MT_HSH_DATZ(4); 664 CVMX_MT_HSH_DATZ(5); 665 CVMX_MT_HSH_DATZ(6); 666 CVMX_ES64(tmp1, ((64 + 16) << 3)); 667 CVMX_MT_HSH_STARTMD5(tmp1); 668 669 /* save the HMAC */ 670 IOV_INIT(iov, data, data_i, data_l); 671 while (icv_off > 0) { 672 IOV_CONSUME(iov, data, data_i, data_l); 673 icv_off -= 8; 674 } 675 CVMX_MF_HSH_IV(*data, 0); 676 IOV_CONSUME(iov, data, data_i, data_l); 677 CVMX_MF_HSH_IV(tmp1, 1); 678 *(uint32_t *)data = (uint32_t) (tmp1 >> 32); 679 680 return 0; 681 } 682 683 /****************************************************************************/ 684 /* SHA1 */ 685 686 int 687 octo_null_sha1_encrypt( 688 struct octo_sess *od, 689 struct iovec *iov, size_t iovcnt, size_t iovlen, 690 int auth_off, int auth_len, 691 int crypt_off, int crypt_len, 692 int icv_off, uint8_t *ivp) 693 { 694 int next = 0; 695 uint64_t *data; 696 uint64_t tmp1, tmp2, tmp3; 697 int data_i, data_l, alen = auth_len; 698 699 dprintf("%s()\n", __func__); 700 701 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || 702 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { 703 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 704 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 705 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 706 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 707 return -EINVAL; 708 } 709 710 IOV_INIT(iov, data, data_i, data_l); 711 712 /* Load SHA1 IV */ 713 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 714 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 715 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 716 717 while (auth_off > 0) { 718 IOV_CONSUME(iov, data, data_i, data_l); 719 auth_off -= 8; 720 } 721 722 while (auth_len > 0) { 723 CVM_LOAD_SHA_UNIT(*data, next); 724 auth_len -= 8; 725 IOV_CONSUME(iov, data, data_i, data_l); 726 } 727 728 /* finish the hash */ 729 CVMX_PREFETCH0(od->octo_hmouter); 730 #if 0 731 if (__predict_false(inplen)) { 732 uint64_t tmp = 0; 733 uint8_t *p = (uint8_t *) & tmp; 734 p[inplen] = 0x80; 735 do { 736 inplen--; 737 p[inplen] = ((uint8_t *) data)[inplen]; 738 } while (inplen); 739 CVM_LOAD_MD5_UNIT(tmp, next); 740 } else { 741 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 742 } 743 #else 744 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 745 #endif 746 747 /* Finish Inner hash */ 748 while (next != 7) { 749 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 750 } 751 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 752 753 /* Get the inner hash of HMAC */ 754 CVMX_MF_HSH_IV(tmp1, 0); 755 CVMX_MF_HSH_IV(tmp2, 1); 756 tmp3 = 0; 757 CVMX_MF_HSH_IV(tmp3, 2); 758 759 /* Initialize hash unit */ 760 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 761 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 762 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 763 764 CVMX_MT_HSH_DAT(tmp1, 0); 765 CVMX_MT_HSH_DAT(tmp2, 1); 766 tmp3 |= 0x0000000080000000; 767 CVMX_MT_HSH_DAT(tmp3, 2); 768 CVMX_MT_HSH_DATZ(3); 769 CVMX_MT_HSH_DATZ(4); 770 CVMX_MT_HSH_DATZ(5); 771 CVMX_MT_HSH_DATZ(6); 772 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 773 774 /* save the HMAC */ 775 IOV_INIT(iov, data, data_i, data_l); 776 while (icv_off > 0) { 777 IOV_CONSUME(iov, data, data_i, data_l); 778 icv_off -= 8; 779 } 780 CVMX_MF_HSH_IV(*data, 0); 781 IOV_CONSUME(iov, data, data_i, data_l); 782 CVMX_MF_HSH_IV(tmp1, 1); 783 *(uint32_t *)data = (uint32_t) (tmp1 >> 32); 784 785 return 0; 786 } 787 788 /****************************************************************************/ 789 /* DES MD5 */ 790 791 int 792 octo_des_cbc_md5_encrypt( 793 struct octo_sess *od, 794 struct iovec *iov, size_t iovcnt, size_t iovlen, 795 int auth_off, int auth_len, 796 int crypt_off, int crypt_len, 797 int icv_off, uint8_t *ivp) 798 { 799 int next = 0; 800 union { 801 uint32_t data32[2]; 802 uint64_t data64[1]; 803 } mydata; 804 uint64_t *data = &mydata.data64[0]; 805 uint32_t *data32; 806 uint64_t tmp1, tmp2; 807 int data_i, data_l, alen = auth_len; 808 809 dprintf("%s()\n", __func__); 810 811 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 812 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 813 (crypt_len & 0x7) || 814 (auth_len & 0x7) || 815 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 816 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 817 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 818 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 819 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 820 return -EINVAL; 821 } 822 823 IOV_INIT(iov, data32, data_i, data_l); 824 825 CVMX_PREFETCH0(ivp); 826 CVMX_PREFETCH0(od->octo_enckey); 827 828 /* load 3DES Key */ 829 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 830 if (od->octo_encklen == 24) { 831 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 832 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 833 } else if (od->octo_encklen == 8) { 834 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 835 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 836 } else { 837 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 838 return -EINVAL; 839 } 840 841 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 842 843 /* Load MD5 IV */ 844 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 845 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 846 847 while (crypt_off > 0 && auth_off > 0) { 848 IOV_CONSUME(iov, data32, data_i, data_l); 849 crypt_off -= 4; 850 auth_off -= 4; 851 } 852 853 while (crypt_len > 0 || auth_len > 0) { 854 uint32_t *first = data32; 855 mydata.data32[0] = *first; 856 IOV_CONSUME(iov, data32, data_i, data_l); 857 mydata.data32[1] = *data32; 858 if (crypt_off <= 0) { 859 if (crypt_len > 0) { 860 CVMX_MT_3DES_ENC_CBC(*data); 861 CVMX_MF_3DES_RESULT(*data); 862 crypt_len -= 8; 863 } 864 } else 865 crypt_off -= 8; 866 if (auth_off <= 0) { 867 if (auth_len > 0) { 868 CVM_LOAD_MD5_UNIT(*data, next); 869 auth_len -= 8; 870 } 871 } else 872 auth_off -= 8; 873 *first = mydata.data32[0]; 874 *data32 = mydata.data32[1]; 875 IOV_CONSUME(iov, data32, data_i, data_l); 876 } 877 878 /* finish the hash */ 879 CVMX_PREFETCH0(od->octo_hmouter); 880 #if 0 881 if (__predict_false(inplen)) { 882 uint64_t tmp = 0; 883 uint8_t *p = (uint8_t *) & tmp; 884 p[inplen] = 0x80; 885 do { 886 inplen--; 887 p[inplen] = ((uint8_t *) data)[inplen]; 888 } while (inplen); 889 CVM_LOAD_MD5_UNIT(tmp, next); 890 } else { 891 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 892 } 893 #else 894 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 895 #endif 896 897 /* Finish Inner hash */ 898 while (next != 7) { 899 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 900 } 901 CVMX_ES64(tmp1, ((alen + 64) << 3)); 902 CVM_LOAD_MD5_UNIT(tmp1, next); 903 904 /* Get the inner hash of HMAC */ 905 CVMX_MF_HSH_IV(tmp1, 0); 906 CVMX_MF_HSH_IV(tmp2, 1); 907 908 /* Initialize hash unit */ 909 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 910 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 911 912 CVMX_MT_HSH_DAT(tmp1, 0); 913 CVMX_MT_HSH_DAT(tmp2, 1); 914 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 915 CVMX_MT_HSH_DATZ(3); 916 CVMX_MT_HSH_DATZ(4); 917 CVMX_MT_HSH_DATZ(5); 918 CVMX_MT_HSH_DATZ(6); 919 CVMX_ES64(tmp1, ((64 + 16) << 3)); 920 CVMX_MT_HSH_STARTMD5(tmp1); 921 922 /* save the HMAC */ 923 IOV_INIT(iov, data32, data_i, data_l); 924 while (icv_off > 0) { 925 IOV_CONSUME(iov, data32, data_i, data_l); 926 icv_off -= 4; 927 } 928 CVMX_MF_HSH_IV(tmp1, 0); 929 *data32 = (uint32_t) (tmp1 >> 32); 930 IOV_CONSUME(iov, data32, data_i, data_l); 931 *data32 = (uint32_t) tmp1; 932 IOV_CONSUME(iov, data32, data_i, data_l); 933 CVMX_MF_HSH_IV(tmp1, 1); 934 *data32 = (uint32_t) (tmp1 >> 32); 935 936 return 0; 937 } 938 939 int 940 octo_des_cbc_md5_decrypt( 941 struct octo_sess *od, 942 struct iovec *iov, size_t iovcnt, size_t iovlen, 943 int auth_off, int auth_len, 944 int crypt_off, int crypt_len, 945 int icv_off, uint8_t *ivp) 946 { 947 int next = 0; 948 union { 949 uint32_t data32[2]; 950 uint64_t data64[1]; 951 } mydata; 952 uint64_t *data = &mydata.data64[0]; 953 uint32_t *data32; 954 uint64_t tmp1, tmp2; 955 int data_i, data_l, alen = auth_len; 956 957 dprintf("%s()\n", __func__); 958 959 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 960 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 961 (crypt_len & 0x7) || 962 (auth_len & 0x7) || 963 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 964 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 965 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 966 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 967 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 968 return -EINVAL; 969 } 970 971 IOV_INIT(iov, data32, data_i, data_l); 972 973 CVMX_PREFETCH0(ivp); 974 CVMX_PREFETCH0(od->octo_enckey); 975 976 /* load 3DES Key */ 977 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 978 if (od->octo_encklen == 24) { 979 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 980 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 981 } else if (od->octo_encklen == 8) { 982 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 983 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 984 } else { 985 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 986 return -EINVAL; 987 } 988 989 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 990 991 /* Load MD5 IV */ 992 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 993 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 994 995 while (crypt_off > 0 && auth_off > 0) { 996 IOV_CONSUME(iov, data32, data_i, data_l); 997 crypt_off -= 4; 998 auth_off -= 4; 999 } 1000 1001 while (crypt_len > 0 || auth_len > 0) { 1002 uint32_t *first = data32; 1003 mydata.data32[0] = *first; 1004 IOV_CONSUME(iov, data32, data_i, data_l); 1005 mydata.data32[1] = *data32; 1006 if (auth_off <= 0) { 1007 if (auth_len > 0) { 1008 CVM_LOAD_MD5_UNIT(*data, next); 1009 auth_len -= 8; 1010 } 1011 } else 1012 auth_off -= 8; 1013 if (crypt_off <= 0) { 1014 if (crypt_len > 0) { 1015 CVMX_MT_3DES_DEC_CBC(*data); 1016 CVMX_MF_3DES_RESULT(*data); 1017 crypt_len -= 8; 1018 } 1019 } else 1020 crypt_off -= 8; 1021 *first = mydata.data32[0]; 1022 *data32 = mydata.data32[1]; 1023 IOV_CONSUME(iov, data32, data_i, data_l); 1024 } 1025 1026 /* finish the hash */ 1027 CVMX_PREFETCH0(od->octo_hmouter); 1028 #if 0 1029 if (__predict_false(inplen)) { 1030 uint64_t tmp = 0; 1031 uint8_t *p = (uint8_t *) & tmp; 1032 p[inplen] = 0x80; 1033 do { 1034 inplen--; 1035 p[inplen] = ((uint8_t *) data)[inplen]; 1036 } while (inplen); 1037 CVM_LOAD_MD5_UNIT(tmp, next); 1038 } else { 1039 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1040 } 1041 #else 1042 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1043 #endif 1044 1045 /* Finish Inner hash */ 1046 while (next != 7) { 1047 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1048 } 1049 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1050 CVM_LOAD_MD5_UNIT(tmp1, next); 1051 1052 /* Get the inner hash of HMAC */ 1053 CVMX_MF_HSH_IV(tmp1, 0); 1054 CVMX_MF_HSH_IV(tmp2, 1); 1055 1056 /* Initialize hash unit */ 1057 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1058 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1059 1060 CVMX_MT_HSH_DAT(tmp1, 0); 1061 CVMX_MT_HSH_DAT(tmp2, 1); 1062 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1063 CVMX_MT_HSH_DATZ(3); 1064 CVMX_MT_HSH_DATZ(4); 1065 CVMX_MT_HSH_DATZ(5); 1066 CVMX_MT_HSH_DATZ(6); 1067 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1068 CVMX_MT_HSH_STARTMD5(tmp1); 1069 1070 /* save the HMAC */ 1071 IOV_INIT(iov, data32, data_i, data_l); 1072 while (icv_off > 0) { 1073 IOV_CONSUME(iov, data32, data_i, data_l); 1074 icv_off -= 4; 1075 } 1076 CVMX_MF_HSH_IV(tmp1, 0); 1077 *data32 = (uint32_t) (tmp1 >> 32); 1078 IOV_CONSUME(iov, data32, data_i, data_l); 1079 *data32 = (uint32_t) tmp1; 1080 IOV_CONSUME(iov, data32, data_i, data_l); 1081 CVMX_MF_HSH_IV(tmp1, 1); 1082 *data32 = (uint32_t) (tmp1 >> 32); 1083 1084 return 0; 1085 } 1086 1087 /****************************************************************************/ 1088 /* DES SHA */ 1089 1090 int 1091 octo_des_cbc_sha1_encrypt( 1092 struct octo_sess *od, 1093 struct iovec *iov, size_t iovcnt, size_t iovlen, 1094 int auth_off, int auth_len, 1095 int crypt_off, int crypt_len, 1096 int icv_off, uint8_t *ivp) 1097 { 1098 int next = 0; 1099 union { 1100 uint32_t data32[2]; 1101 uint64_t data64[1]; 1102 } mydata; 1103 uint64_t *data = &mydata.data64[0]; 1104 uint32_t *data32; 1105 uint64_t tmp1, tmp2, tmp3; 1106 int data_i, data_l, alen = auth_len; 1107 1108 dprintf("%s()\n", __func__); 1109 1110 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1111 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1112 (crypt_len & 0x7) || 1113 (auth_len & 0x7) || 1114 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1115 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1116 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1117 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1118 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1119 return -EINVAL; 1120 } 1121 1122 IOV_INIT(iov, data32, data_i, data_l); 1123 1124 CVMX_PREFETCH0(ivp); 1125 CVMX_PREFETCH0(od->octo_enckey); 1126 1127 /* load 3DES Key */ 1128 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1129 if (od->octo_encklen == 24) { 1130 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1131 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1132 } else if (od->octo_encklen == 8) { 1133 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 1134 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 1135 } else { 1136 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1137 return -EINVAL; 1138 } 1139 1140 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 1141 1142 /* Load SHA1 IV */ 1143 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1144 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1145 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1146 1147 while (crypt_off > 0 && auth_off > 0) { 1148 IOV_CONSUME(iov, data32, data_i, data_l); 1149 crypt_off -= 4; 1150 auth_off -= 4; 1151 } 1152 1153 while (crypt_len > 0 || auth_len > 0) { 1154 uint32_t *first = data32; 1155 mydata.data32[0] = *first; 1156 IOV_CONSUME(iov, data32, data_i, data_l); 1157 mydata.data32[1] = *data32; 1158 if (crypt_off <= 0) { 1159 if (crypt_len > 0) { 1160 CVMX_MT_3DES_ENC_CBC(*data); 1161 CVMX_MF_3DES_RESULT(*data); 1162 crypt_len -= 8; 1163 } 1164 } else 1165 crypt_off -= 8; 1166 if (auth_off <= 0) { 1167 if (auth_len > 0) { 1168 CVM_LOAD_SHA_UNIT(*data, next); 1169 auth_len -= 8; 1170 } 1171 } else 1172 auth_off -= 8; 1173 *first = mydata.data32[0]; 1174 *data32 = mydata.data32[1]; 1175 IOV_CONSUME(iov, data32, data_i, data_l); 1176 } 1177 1178 /* finish the hash */ 1179 CVMX_PREFETCH0(od->octo_hmouter); 1180 #if 0 1181 if (__predict_false(inplen)) { 1182 uint64_t tmp = 0; 1183 uint8_t *p = (uint8_t *) & tmp; 1184 p[inplen] = 0x80; 1185 do { 1186 inplen--; 1187 p[inplen] = ((uint8_t *) data)[inplen]; 1188 } while (inplen); 1189 CVM_LOAD_SHA_UNIT(tmp, next); 1190 } else { 1191 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1192 } 1193 #else 1194 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1195 #endif 1196 1197 /* Finish Inner hash */ 1198 while (next != 7) { 1199 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1200 } 1201 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1202 1203 /* Get the inner hash of HMAC */ 1204 CVMX_MF_HSH_IV(tmp1, 0); 1205 CVMX_MF_HSH_IV(tmp2, 1); 1206 tmp3 = 0; 1207 CVMX_MF_HSH_IV(tmp3, 2); 1208 1209 /* Initialize hash unit */ 1210 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1211 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1212 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1213 1214 CVMX_MT_HSH_DAT(tmp1, 0); 1215 CVMX_MT_HSH_DAT(tmp2, 1); 1216 tmp3 |= 0x0000000080000000; 1217 CVMX_MT_HSH_DAT(tmp3, 2); 1218 CVMX_MT_HSH_DATZ(3); 1219 CVMX_MT_HSH_DATZ(4); 1220 CVMX_MT_HSH_DATZ(5); 1221 CVMX_MT_HSH_DATZ(6); 1222 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1223 1224 /* save the HMAC */ 1225 IOV_INIT(iov, data32, data_i, data_l); 1226 while (icv_off > 0) { 1227 IOV_CONSUME(iov, data32, data_i, data_l); 1228 icv_off -= 4; 1229 } 1230 CVMX_MF_HSH_IV(tmp1, 0); 1231 *data32 = (uint32_t) (tmp1 >> 32); 1232 IOV_CONSUME(iov, data32, data_i, data_l); 1233 *data32 = (uint32_t) tmp1; 1234 IOV_CONSUME(iov, data32, data_i, data_l); 1235 CVMX_MF_HSH_IV(tmp1, 1); 1236 *data32 = (uint32_t) (tmp1 >> 32); 1237 1238 return 0; 1239 } 1240 1241 int 1242 octo_des_cbc_sha1_decrypt( 1243 struct octo_sess *od, 1244 struct iovec *iov, size_t iovcnt, size_t iovlen, 1245 int auth_off, int auth_len, 1246 int crypt_off, int crypt_len, 1247 int icv_off, uint8_t *ivp) 1248 { 1249 int next = 0; 1250 union { 1251 uint32_t data32[2]; 1252 uint64_t data64[1]; 1253 } mydata; 1254 uint64_t *data = &mydata.data64[0]; 1255 uint32_t *data32; 1256 uint64_t tmp1, tmp2, tmp3; 1257 int data_i, data_l, alen = auth_len; 1258 1259 dprintf("%s()\n", __func__); 1260 1261 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1262 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1263 (crypt_len & 0x7) || 1264 (auth_len & 0x7) || 1265 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1266 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1267 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1268 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1269 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1270 return -EINVAL; 1271 } 1272 1273 IOV_INIT(iov, data32, data_i, data_l); 1274 1275 CVMX_PREFETCH0(ivp); 1276 CVMX_PREFETCH0(od->octo_enckey); 1277 1278 /* load 3DES Key */ 1279 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1280 if (od->octo_encklen == 24) { 1281 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1282 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1283 } else if (od->octo_encklen == 8) { 1284 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 1285 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 1286 } else { 1287 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1288 return -EINVAL; 1289 } 1290 1291 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 1292 1293 /* Load SHA1 IV */ 1294 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1295 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1296 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1297 1298 while (crypt_off > 0 && auth_off > 0) { 1299 IOV_CONSUME(iov, data32, data_i, data_l); 1300 crypt_off -= 4; 1301 auth_off -= 4; 1302 } 1303 1304 while (crypt_len > 0 || auth_len > 0) { 1305 uint32_t *first = data32; 1306 mydata.data32[0] = *first; 1307 IOV_CONSUME(iov, data32, data_i, data_l); 1308 mydata.data32[1] = *data32; 1309 if (auth_off <= 0) { 1310 if (auth_len > 0) { 1311 CVM_LOAD_SHA_UNIT(*data, next); 1312 auth_len -= 8; 1313 } 1314 } else 1315 auth_off -= 8; 1316 if (crypt_off <= 0) { 1317 if (crypt_len > 0) { 1318 CVMX_MT_3DES_DEC_CBC(*data); 1319 CVMX_MF_3DES_RESULT(*data); 1320 crypt_len -= 8; 1321 } 1322 } else 1323 crypt_off -= 8; 1324 *first = mydata.data32[0]; 1325 *data32 = mydata.data32[1]; 1326 IOV_CONSUME(iov, data32, data_i, data_l); 1327 } 1328 1329 /* finish the hash */ 1330 CVMX_PREFETCH0(od->octo_hmouter); 1331 #if 0 1332 if (__predict_false(inplen)) { 1333 uint64_t tmp = 0; 1334 uint8_t *p = (uint8_t *) & tmp; 1335 p[inplen] = 0x80; 1336 do { 1337 inplen--; 1338 p[inplen] = ((uint8_t *) data)[inplen]; 1339 } while (inplen); 1340 CVM_LOAD_SHA_UNIT(tmp, next); 1341 } else { 1342 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1343 } 1344 #else 1345 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1346 #endif 1347 1348 /* Finish Inner hash */ 1349 while (next != 7) { 1350 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1351 } 1352 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1353 1354 /* Get the inner hash of HMAC */ 1355 CVMX_MF_HSH_IV(tmp1, 0); 1356 CVMX_MF_HSH_IV(tmp2, 1); 1357 tmp3 = 0; 1358 CVMX_MF_HSH_IV(tmp3, 2); 1359 1360 /* Initialize hash unit */ 1361 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1362 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1363 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1364 1365 CVMX_MT_HSH_DAT(tmp1, 0); 1366 CVMX_MT_HSH_DAT(tmp2, 1); 1367 tmp3 |= 0x0000000080000000; 1368 CVMX_MT_HSH_DAT(tmp3, 2); 1369 CVMX_MT_HSH_DATZ(3); 1370 CVMX_MT_HSH_DATZ(4); 1371 CVMX_MT_HSH_DATZ(5); 1372 CVMX_MT_HSH_DATZ(6); 1373 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1374 /* save the HMAC */ 1375 IOV_INIT(iov, data32, data_i, data_l); 1376 while (icv_off > 0) { 1377 IOV_CONSUME(iov, data32, data_i, data_l); 1378 icv_off -= 4; 1379 } 1380 CVMX_MF_HSH_IV(tmp1, 0); 1381 *data32 = (uint32_t) (tmp1 >> 32); 1382 IOV_CONSUME(iov, data32, data_i, data_l); 1383 *data32 = (uint32_t) tmp1; 1384 IOV_CONSUME(iov, data32, data_i, data_l); 1385 CVMX_MF_HSH_IV(tmp1, 1); 1386 *data32 = (uint32_t) (tmp1 >> 32); 1387 1388 return 0; 1389 } 1390 1391 /****************************************************************************/ 1392 /* AES MD5 */ 1393 1394 int 1395 octo_aes_cbc_md5_encrypt( 1396 struct octo_sess *od, 1397 struct iovec *iov, size_t iovcnt, size_t iovlen, 1398 int auth_off, int auth_len, 1399 int crypt_off, int crypt_len, 1400 int icv_off, uint8_t *ivp) 1401 { 1402 int next = 0; 1403 union { 1404 uint32_t data32[2]; 1405 uint64_t data64[1]; 1406 } mydata[2]; 1407 uint64_t *pdata = &mydata[0].data64[0]; 1408 uint64_t *data = &mydata[1].data64[0]; 1409 uint32_t *data32; 1410 uint64_t tmp1, tmp2; 1411 int data_i, data_l, alen = auth_len; 1412 1413 dprintf("%s()\n", __func__); 1414 1415 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1416 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1417 (crypt_len & 0x7) || 1418 (auth_len & 0x7) || 1419 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1420 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1421 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1422 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1423 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1424 return -EINVAL; 1425 } 1426 1427 IOV_INIT(iov, data32, data_i, data_l); 1428 1429 CVMX_PREFETCH0(ivp); 1430 CVMX_PREFETCH0(od->octo_enckey); 1431 1432 /* load AES Key */ 1433 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1434 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1435 1436 if (od->octo_encklen == 16) { 1437 CVMX_MT_AES_KEY(0x0, 2); 1438 CVMX_MT_AES_KEY(0x0, 3); 1439 } else if (od->octo_encklen == 24) { 1440 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1441 CVMX_MT_AES_KEY(0x0, 3); 1442 } else if (od->octo_encklen == 32) { 1443 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1444 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1445 } else { 1446 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1447 return -EINVAL; 1448 } 1449 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1450 1451 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1452 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1453 1454 /* Load MD5 IV */ 1455 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1456 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1457 1458 while (crypt_off > 0 && auth_off > 0) { 1459 IOV_CONSUME(iov, data32, data_i, data_l); 1460 crypt_off -= 4; 1461 auth_off -= 4; 1462 } 1463 1464 while (crypt_len > 0 || auth_len > 0) { 1465 uint32_t *pdata32[3]; 1466 1467 pdata32[0] = data32; 1468 mydata[0].data32[0] = *data32; 1469 IOV_CONSUME(iov, data32, data_i, data_l); 1470 1471 pdata32[1] = data32; 1472 mydata[0].data32[1] = *data32; 1473 IOV_CONSUME(iov, data32, data_i, data_l); 1474 1475 pdata32[2] = data32; 1476 mydata[1].data32[0] = *data32; 1477 IOV_CONSUME(iov, data32, data_i, data_l); 1478 1479 mydata[1].data32[1] = *data32; 1480 1481 1482 if (crypt_off <= 0) { 1483 if (crypt_len > 0) { 1484 CVMX_MT_AES_ENC_CBC0(*pdata); 1485 CVMX_MT_AES_ENC_CBC1(*data); 1486 CVMX_MF_AES_RESULT(*pdata, 0); 1487 CVMX_MF_AES_RESULT(*data, 1); 1488 crypt_len -= 16; 1489 } 1490 } else 1491 crypt_off -= 16; 1492 1493 if (auth_off <= 0) { 1494 if (auth_len > 0) { 1495 CVM_LOAD_MD5_UNIT(*pdata, next); 1496 CVM_LOAD_MD5_UNIT(*data, next); 1497 auth_len -= 16; 1498 } 1499 } else 1500 auth_off -= 16; 1501 1502 *pdata32[0] = mydata[0].data32[0]; 1503 *pdata32[1] = mydata[0].data32[1]; 1504 *pdata32[2] = mydata[1].data32[0]; 1505 *data32 = mydata[1].data32[1]; 1506 1507 IOV_CONSUME(iov, data32, data_i, data_l); 1508 } 1509 1510 /* finish the hash */ 1511 CVMX_PREFETCH0(od->octo_hmouter); 1512 #if 0 1513 if (__predict_false(inplen)) { 1514 uint64_t tmp = 0; 1515 uint8_t *p = (uint8_t *) & tmp; 1516 p[inplen] = 0x80; 1517 do { 1518 inplen--; 1519 p[inplen] = ((uint8_t *) data)[inplen]; 1520 } while (inplen); 1521 CVM_LOAD_MD5_UNIT(tmp, next); 1522 } else { 1523 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1524 } 1525 #else 1526 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1527 #endif 1528 1529 /* Finish Inner hash */ 1530 while (next != 7) { 1531 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1532 } 1533 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1534 CVM_LOAD_MD5_UNIT(tmp1, next); 1535 1536 /* Get the inner hash of HMAC */ 1537 CVMX_MF_HSH_IV(tmp1, 0); 1538 CVMX_MF_HSH_IV(tmp2, 1); 1539 1540 /* Initialize hash unit */ 1541 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1542 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1543 1544 CVMX_MT_HSH_DAT(tmp1, 0); 1545 CVMX_MT_HSH_DAT(tmp2, 1); 1546 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1547 CVMX_MT_HSH_DATZ(3); 1548 CVMX_MT_HSH_DATZ(4); 1549 CVMX_MT_HSH_DATZ(5); 1550 CVMX_MT_HSH_DATZ(6); 1551 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1552 CVMX_MT_HSH_STARTMD5(tmp1); 1553 1554 /* save the HMAC */ 1555 IOV_INIT(iov, data32, data_i, data_l); 1556 while (icv_off > 0) { 1557 IOV_CONSUME(iov, data32, data_i, data_l); 1558 icv_off -= 4; 1559 } 1560 CVMX_MF_HSH_IV(tmp1, 0); 1561 *data32 = (uint32_t) (tmp1 >> 32); 1562 IOV_CONSUME(iov, data32, data_i, data_l); 1563 *data32 = (uint32_t) tmp1; 1564 IOV_CONSUME(iov, data32, data_i, data_l); 1565 CVMX_MF_HSH_IV(tmp1, 1); 1566 *data32 = (uint32_t) (tmp1 >> 32); 1567 1568 return 0; 1569 } 1570 1571 int 1572 octo_aes_cbc_md5_decrypt( 1573 struct octo_sess *od, 1574 struct iovec *iov, size_t iovcnt, size_t iovlen, 1575 int auth_off, int auth_len, 1576 int crypt_off, int crypt_len, 1577 int icv_off, uint8_t *ivp) 1578 { 1579 int next = 0; 1580 union { 1581 uint32_t data32[2]; 1582 uint64_t data64[1]; 1583 } mydata[2]; 1584 uint64_t *pdata = &mydata[0].data64[0]; 1585 uint64_t *data = &mydata[1].data64[0]; 1586 uint32_t *data32; 1587 uint64_t tmp1, tmp2; 1588 int data_i, data_l, alen = auth_len; 1589 1590 dprintf("%s()\n", __func__); 1591 1592 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1593 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1594 (crypt_len & 0x7) || 1595 (auth_len & 0x7) || 1596 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1597 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1598 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1599 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1600 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1601 return -EINVAL; 1602 } 1603 1604 IOV_INIT(iov, data32, data_i, data_l); 1605 1606 CVMX_PREFETCH0(ivp); 1607 CVMX_PREFETCH0(od->octo_enckey); 1608 1609 /* load AES Key */ 1610 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1611 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1612 1613 if (od->octo_encklen == 16) { 1614 CVMX_MT_AES_KEY(0x0, 2); 1615 CVMX_MT_AES_KEY(0x0, 3); 1616 } else if (od->octo_encklen == 24) { 1617 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1618 CVMX_MT_AES_KEY(0x0, 3); 1619 } else if (od->octo_encklen == 32) { 1620 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1621 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1622 } else { 1623 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1624 return -EINVAL; 1625 } 1626 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1627 1628 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1629 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1630 1631 /* Load MD5 IV */ 1632 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1633 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1634 1635 while (crypt_off > 0 && auth_off > 0) { 1636 IOV_CONSUME(iov, data32, data_i, data_l); 1637 crypt_off -= 4; 1638 auth_off -= 4; 1639 } 1640 1641 while (crypt_len > 0 || auth_len > 0) { 1642 uint32_t *pdata32[3]; 1643 1644 pdata32[0] = data32; 1645 mydata[0].data32[0] = *data32; 1646 IOV_CONSUME(iov, data32, data_i, data_l); 1647 pdata32[1] = data32; 1648 mydata[0].data32[1] = *data32; 1649 IOV_CONSUME(iov, data32, data_i, data_l); 1650 pdata32[2] = data32; 1651 mydata[1].data32[0] = *data32; 1652 IOV_CONSUME(iov, data32, data_i, data_l); 1653 mydata[1].data32[1] = *data32; 1654 1655 if (auth_off <= 0) { 1656 if (auth_len > 0) { 1657 CVM_LOAD_MD5_UNIT(*pdata, next); 1658 CVM_LOAD_MD5_UNIT(*data, next); 1659 auth_len -= 16; 1660 } 1661 } else 1662 auth_off -= 16; 1663 1664 if (crypt_off <= 0) { 1665 if (crypt_len > 0) { 1666 CVMX_MT_AES_DEC_CBC0(*pdata); 1667 CVMX_MT_AES_DEC_CBC1(*data); 1668 CVMX_MF_AES_RESULT(*pdata, 0); 1669 CVMX_MF_AES_RESULT(*data, 1); 1670 crypt_len -= 16; 1671 } 1672 } else 1673 crypt_off -= 16; 1674 1675 *pdata32[0] = mydata[0].data32[0]; 1676 *pdata32[1] = mydata[0].data32[1]; 1677 *pdata32[2] = mydata[1].data32[0]; 1678 *data32 = mydata[1].data32[1]; 1679 1680 IOV_CONSUME(iov, data32, data_i, data_l); 1681 } 1682 1683 /* finish the hash */ 1684 CVMX_PREFETCH0(od->octo_hmouter); 1685 #if 0 1686 if (__predict_false(inplen)) { 1687 uint64_t tmp = 0; 1688 uint8_t *p = (uint8_t *) & tmp; 1689 p[inplen] = 0x80; 1690 do { 1691 inplen--; 1692 p[inplen] = ((uint8_t *) data)[inplen]; 1693 } while (inplen); 1694 CVM_LOAD_MD5_UNIT(tmp, next); 1695 } else { 1696 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1697 } 1698 #else 1699 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1700 #endif 1701 1702 /* Finish Inner hash */ 1703 while (next != 7) { 1704 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1705 } 1706 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1707 CVM_LOAD_MD5_UNIT(tmp1, next); 1708 1709 /* Get the inner hash of HMAC */ 1710 CVMX_MF_HSH_IV(tmp1, 0); 1711 CVMX_MF_HSH_IV(tmp2, 1); 1712 1713 /* Initialize hash unit */ 1714 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1715 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1716 1717 CVMX_MT_HSH_DAT(tmp1, 0); 1718 CVMX_MT_HSH_DAT(tmp2, 1); 1719 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1720 CVMX_MT_HSH_DATZ(3); 1721 CVMX_MT_HSH_DATZ(4); 1722 CVMX_MT_HSH_DATZ(5); 1723 CVMX_MT_HSH_DATZ(6); 1724 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1725 CVMX_MT_HSH_STARTMD5(tmp1); 1726 1727 /* save the HMAC */ 1728 IOV_INIT(iov, data32, data_i, data_l); 1729 while (icv_off > 0) { 1730 IOV_CONSUME(iov, data32, data_i, data_l); 1731 icv_off -= 4; 1732 } 1733 CVMX_MF_HSH_IV(tmp1, 0); 1734 *data32 = (uint32_t) (tmp1 >> 32); 1735 IOV_CONSUME(iov, data32, data_i, data_l); 1736 *data32 = (uint32_t) tmp1; 1737 IOV_CONSUME(iov, data32, data_i, data_l); 1738 CVMX_MF_HSH_IV(tmp1, 1); 1739 *data32 = (uint32_t) (tmp1 >> 32); 1740 1741 return 0; 1742 } 1743 1744 /****************************************************************************/ 1745 /* AES SHA1 */ 1746 1747 int 1748 octo_aes_cbc_sha1_encrypt( 1749 struct octo_sess *od, 1750 struct iovec *iov, size_t iovcnt, size_t iovlen, 1751 int auth_off, int auth_len, 1752 int crypt_off, int crypt_len, 1753 int icv_off, uint8_t *ivp) 1754 { 1755 int next = 0; 1756 union { 1757 uint32_t data32[2]; 1758 uint64_t data64[1]; 1759 } mydata[2]; 1760 uint64_t *pdata = &mydata[0].data64[0]; 1761 uint64_t *data = &mydata[1].data64[0]; 1762 uint32_t *data32; 1763 uint64_t tmp1, tmp2, tmp3; 1764 int data_i, data_l, alen = auth_len; 1765 1766 dprintf("%s()\n", __func__); 1767 1768 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1769 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1770 (crypt_len & 0x7) || 1771 (auth_len & 0x7) || 1772 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1773 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1774 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1775 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1776 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1777 return -EINVAL; 1778 } 1779 1780 IOV_INIT(iov, data32, data_i, data_l); 1781 1782 CVMX_PREFETCH0(ivp); 1783 CVMX_PREFETCH0(od->octo_enckey); 1784 1785 /* load AES Key */ 1786 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1787 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1788 1789 if (od->octo_encklen == 16) { 1790 CVMX_MT_AES_KEY(0x0, 2); 1791 CVMX_MT_AES_KEY(0x0, 3); 1792 } else if (od->octo_encklen == 24) { 1793 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1794 CVMX_MT_AES_KEY(0x0, 3); 1795 } else if (od->octo_encklen == 32) { 1796 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1797 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1798 } else { 1799 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1800 return -EINVAL; 1801 } 1802 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1803 1804 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1805 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1806 1807 /* Load SHA IV */ 1808 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1809 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1810 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1811 1812 while (crypt_off > 0 && auth_off > 0) { 1813 IOV_CONSUME(iov, data32, data_i, data_l); 1814 crypt_off -= 4; 1815 auth_off -= 4; 1816 } 1817 1818 while (crypt_len > 0 || auth_len > 0) { 1819 uint32_t *pdata32[3]; 1820 1821 pdata32[0] = data32; 1822 mydata[0].data32[0] = *data32; 1823 IOV_CONSUME(iov, data32, data_i, data_l); 1824 pdata32[1] = data32; 1825 mydata[0].data32[1] = *data32; 1826 IOV_CONSUME(iov, data32, data_i, data_l); 1827 pdata32[2] = data32; 1828 mydata[1].data32[0] = *data32; 1829 IOV_CONSUME(iov, data32, data_i, data_l); 1830 mydata[1].data32[1] = *data32; 1831 1832 1833 if (crypt_off <= 0) { 1834 if (crypt_len > 0) { 1835 CVMX_MT_AES_ENC_CBC0(*pdata); 1836 CVMX_MT_AES_ENC_CBC1(*data); 1837 CVMX_MF_AES_RESULT(*pdata, 0); 1838 CVMX_MF_AES_RESULT(*data, 1); 1839 crypt_len -= 16; 1840 } 1841 } else 1842 crypt_off -= 16; 1843 1844 if (auth_off <= 0) { 1845 if (auth_len > 0) { 1846 CVM_LOAD_SHA_UNIT(*pdata, next); 1847 CVM_LOAD_SHA_UNIT(*data, next); 1848 auth_len -= 16; 1849 } 1850 } else 1851 auth_off -= 16; 1852 1853 *pdata32[0] = mydata[0].data32[0]; 1854 *pdata32[1] = mydata[0].data32[1]; 1855 *pdata32[2] = mydata[1].data32[0]; 1856 *data32 = mydata[1].data32[1]; 1857 1858 IOV_CONSUME(iov, data32, data_i, data_l); 1859 } 1860 1861 /* finish the hash */ 1862 CVMX_PREFETCH0(od->octo_hmouter); 1863 #if 0 1864 if (__predict_false(inplen)) { 1865 uint64_t tmp = 0; 1866 uint8_t *p = (uint8_t *) & tmp; 1867 p[inplen] = 0x80; 1868 do { 1869 inplen--; 1870 p[inplen] = ((uint8_t *) data)[inplen]; 1871 } while (inplen); 1872 CVM_LOAD_SHA_UNIT(tmp, next); 1873 } else { 1874 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1875 } 1876 #else 1877 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1878 #endif 1879 1880 /* Finish Inner hash */ 1881 while (next != 7) { 1882 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1883 } 1884 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1885 1886 /* Get the inner hash of HMAC */ 1887 CVMX_MF_HSH_IV(tmp1, 0); 1888 CVMX_MF_HSH_IV(tmp2, 1); 1889 tmp3 = 0; 1890 CVMX_MF_HSH_IV(tmp3, 2); 1891 1892 /* Initialize hash unit */ 1893 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1894 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1895 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1896 1897 CVMX_MT_HSH_DAT(tmp1, 0); 1898 CVMX_MT_HSH_DAT(tmp2, 1); 1899 tmp3 |= 0x0000000080000000; 1900 CVMX_MT_HSH_DAT(tmp3, 2); 1901 CVMX_MT_HSH_DATZ(3); 1902 CVMX_MT_HSH_DATZ(4); 1903 CVMX_MT_HSH_DATZ(5); 1904 CVMX_MT_HSH_DATZ(6); 1905 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1906 1907 /* finish the hash */ 1908 CVMX_PREFETCH0(od->octo_hmouter); 1909 #if 0 1910 if (__predict_false(inplen)) { 1911 uint64_t tmp = 0; 1912 uint8_t *p = (uint8_t *) & tmp; 1913 p[inplen] = 0x80; 1914 do { 1915 inplen--; 1916 p[inplen] = ((uint8_t *) data)[inplen]; 1917 } while (inplen); 1918 CVM_LOAD_MD5_UNIT(tmp, next); 1919 } else { 1920 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1921 } 1922 #else 1923 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1924 #endif 1925 1926 /* save the HMAC */ 1927 IOV_INIT(iov, data32, data_i, data_l); 1928 while (icv_off > 0) { 1929 IOV_CONSUME(iov, data32, data_i, data_l); 1930 icv_off -= 4; 1931 } 1932 CVMX_MF_HSH_IV(tmp1, 0); 1933 *data32 = (uint32_t) (tmp1 >> 32); 1934 IOV_CONSUME(iov, data32, data_i, data_l); 1935 *data32 = (uint32_t) tmp1; 1936 IOV_CONSUME(iov, data32, data_i, data_l); 1937 CVMX_MF_HSH_IV(tmp1, 1); 1938 *data32 = (uint32_t) (tmp1 >> 32); 1939 1940 return 0; 1941 } 1942 1943 int 1944 octo_aes_cbc_sha1_decrypt( 1945 struct octo_sess *od, 1946 struct iovec *iov, size_t iovcnt, size_t iovlen, 1947 int auth_off, int auth_len, 1948 int crypt_off, int crypt_len, 1949 int icv_off, uint8_t *ivp) 1950 { 1951 int next = 0; 1952 union { 1953 uint32_t data32[2]; 1954 uint64_t data64[1]; 1955 } mydata[2]; 1956 uint64_t *pdata = &mydata[0].data64[0]; 1957 uint64_t *data = &mydata[1].data64[0]; 1958 uint32_t *data32; 1959 uint64_t tmp1, tmp2, tmp3; 1960 int data_i, data_l, alen = auth_len; 1961 1962 dprintf("%s()\n", __func__); 1963 1964 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1965 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1966 (crypt_len & 0x7) || 1967 (auth_len & 0x7) || 1968 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1969 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1970 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1971 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1972 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1973 return -EINVAL; 1974 } 1975 1976 IOV_INIT(iov, data32, data_i, data_l); 1977 1978 CVMX_PREFETCH0(ivp); 1979 CVMX_PREFETCH0(od->octo_enckey); 1980 1981 /* load AES Key */ 1982 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1983 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1984 1985 if (od->octo_encklen == 16) { 1986 CVMX_MT_AES_KEY(0x0, 2); 1987 CVMX_MT_AES_KEY(0x0, 3); 1988 } else if (od->octo_encklen == 24) { 1989 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1990 CVMX_MT_AES_KEY(0x0, 3); 1991 } else if (od->octo_encklen == 32) { 1992 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1993 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1994 } else { 1995 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1996 return -EINVAL; 1997 } 1998 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1999 2000 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 2001 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 2002 2003 /* Load MD5 IV */ 2004 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 2005 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 2006 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 2007 2008 while (crypt_off > 0 && auth_off > 0) { 2009 IOV_CONSUME(iov, data32, data_i, data_l); 2010 crypt_off -= 4; 2011 auth_off -= 4; 2012 } 2013 2014 while (crypt_len > 0 || auth_len > 0) { 2015 uint32_t *pdata32[3]; 2016 2017 pdata32[0] = data32; 2018 mydata[0].data32[0] = *data32; 2019 IOV_CONSUME(iov, data32, data_i, data_l); 2020 pdata32[1] = data32; 2021 mydata[0].data32[1] = *data32; 2022 IOV_CONSUME(iov, data32, data_i, data_l); 2023 pdata32[2] = data32; 2024 mydata[1].data32[0] = *data32; 2025 IOV_CONSUME(iov, data32, data_i, data_l); 2026 mydata[1].data32[1] = *data32; 2027 2028 if (auth_off <= 0) { 2029 if (auth_len > 0) { 2030 CVM_LOAD_SHA_UNIT(*pdata, next); 2031 CVM_LOAD_SHA_UNIT(*data, next); 2032 auth_len -= 16; 2033 } 2034 } else 2035 auth_off -= 16; 2036 2037 if (crypt_off <= 0) { 2038 if (crypt_len > 0) { 2039 CVMX_MT_AES_DEC_CBC0(*pdata); 2040 CVMX_MT_AES_DEC_CBC1(*data); 2041 CVMX_MF_AES_RESULT(*pdata, 0); 2042 CVMX_MF_AES_RESULT(*data, 1); 2043 crypt_len -= 16; 2044 } 2045 } else 2046 crypt_off -= 16; 2047 2048 *pdata32[0] = mydata[0].data32[0]; 2049 *pdata32[1] = mydata[0].data32[1]; 2050 *pdata32[2] = mydata[1].data32[0]; 2051 *data32 = mydata[1].data32[1]; 2052 2053 IOV_CONSUME(iov, data32, data_i, data_l); 2054 } 2055 2056 /* finish the hash */ 2057 CVMX_PREFETCH0(od->octo_hmouter); 2058 #if 0 2059 if (__predict_false(inplen)) { 2060 uint64_t tmp = 0; 2061 uint8_t *p = (uint8_t *) & tmp; 2062 p[inplen] = 0x80; 2063 do { 2064 inplen--; 2065 p[inplen] = ((uint8_t *) data)[inplen]; 2066 } while (inplen); 2067 CVM_LOAD_SHA_UNIT(tmp, next); 2068 } else { 2069 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 2070 } 2071 #else 2072 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 2073 #endif 2074 2075 /* Finish Inner hash */ 2076 while (next != 7) { 2077 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 2078 } 2079 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 2080 2081 /* Get the inner hash of HMAC */ 2082 CVMX_MF_HSH_IV(tmp1, 0); 2083 CVMX_MF_HSH_IV(tmp2, 1); 2084 tmp3 = 0; 2085 CVMX_MF_HSH_IV(tmp3, 2); 2086 2087 /* Initialize hash unit */ 2088 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 2089 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 2090 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 2091 2092 CVMX_MT_HSH_DAT(tmp1, 0); 2093 CVMX_MT_HSH_DAT(tmp2, 1); 2094 tmp3 |= 0x0000000080000000; 2095 CVMX_MT_HSH_DAT(tmp3, 2); 2096 CVMX_MT_HSH_DATZ(3); 2097 CVMX_MT_HSH_DATZ(4); 2098 CVMX_MT_HSH_DATZ(5); 2099 CVMX_MT_HSH_DATZ(6); 2100 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 2101 2102 /* finish the hash */ 2103 CVMX_PREFETCH0(od->octo_hmouter); 2104 #if 0 2105 if (__predict_false(inplen)) { 2106 uint64_t tmp = 0; 2107 uint8_t *p = (uint8_t *) & tmp; 2108 p[inplen] = 0x80; 2109 do { 2110 inplen--; 2111 p[inplen] = ((uint8_t *) data)[inplen]; 2112 } while (inplen); 2113 CVM_LOAD_MD5_UNIT(tmp, next); 2114 } else { 2115 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 2116 } 2117 #else 2118 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 2119 #endif 2120 2121 /* save the HMAC */ 2122 IOV_INIT(iov, data32, data_i, data_l); 2123 while (icv_off > 0) { 2124 IOV_CONSUME(iov, data32, data_i, data_l); 2125 icv_off -= 4; 2126 } 2127 CVMX_MF_HSH_IV(tmp1, 0); 2128 *data32 = (uint32_t) (tmp1 >> 32); 2129 IOV_CONSUME(iov, data32, data_i, data_l); 2130 *data32 = (uint32_t) tmp1; 2131 IOV_CONSUME(iov, data32, data_i, data_l); 2132 CVMX_MF_HSH_IV(tmp1, 1); 2133 *data32 = (uint32_t) (tmp1 >> 32); 2134 2135 return 0; 2136 } 2137 2138 /****************************************************************************/
Cache object: f4a1cbb9ab249d8dec15fc5d7363e93f
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]