1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Netflix Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/counter.h>
35 #include <sys/endian.h>
36 #include <sys/kernel.h>
37 #include <sys/ktls.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/sysctl.h>
43 #include <sys/uio.h>
44 #include <opencrypto/cryptodev.h>
45
46 struct ocf_session {
47 crypto_session_t sid;
48 crypto_session_t mac_sid;
49 int mac_len;
50 struct mtx lock;
51 bool implicit_iv;
52
53 /* Only used for TLS 1.0 with the implicit IV. */
54 #ifdef INVARIANTS
55 bool in_progress;
56 uint64_t next_seqno;
57 #endif
58 char iv[AES_BLOCK_LEN];
59 };
60
61 struct ocf_operation {
62 struct ocf_session *os;
63 bool done;
64 };
65
66 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
67
68 SYSCTL_DECL(_kern_ipc_tls);
69 SYSCTL_DECL(_kern_ipc_tls_stats);
70
71 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf,
72 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
73 "Kernel TLS offload via OCF stats");
74
75 static counter_u64_t ocf_tls10_cbc_crypts;
76 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls10_cbc_crypts,
77 CTLFLAG_RD, &ocf_tls10_cbc_crypts,
78 "Total number of OCF TLS 1.0 CBC encryption operations");
79
80 static counter_u64_t ocf_tls11_cbc_crypts;
81 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_crypts,
82 CTLFLAG_RD, &ocf_tls11_cbc_crypts,
83 "Total number of OCF TLS 1.1/1.2 CBC encryption operations");
84
85 static counter_u64_t ocf_tls12_gcm_crypts;
86 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
87 CTLFLAG_RD, &ocf_tls12_gcm_crypts,
88 "Total number of OCF TLS 1.2 GCM encryption operations");
89
90 static counter_u64_t ocf_tls13_gcm_crypts;
91 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
92 CTLFLAG_RD, &ocf_tls13_gcm_crypts,
93 "Total number of OCF TLS 1.3 GCM encryption operations");
94
95 static counter_u64_t ocf_inplace;
96 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
97 CTLFLAG_RD, &ocf_inplace,
98 "Total number of OCF in-place operations");
99
100 static counter_u64_t ocf_separate_output;
101 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
102 CTLFLAG_RD, &ocf_separate_output,
103 "Total number of OCF operations with a separate output buffer");
104
105 static counter_u64_t ocf_retries;
106 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
107 &ocf_retries,
108 "Number of OCF encryption operation retries");
109
110 static int
111 ktls_ocf_callback(struct cryptop *crp)
112 {
113 struct ocf_operation *oo;
114
115 oo = crp->crp_opaque;
116 mtx_lock(&oo->os->lock);
117 oo->done = true;
118 mtx_unlock(&oo->os->lock);
119 wakeup(oo);
120 return (0);
121 }
122
123 static int
124 ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp)
125 {
126 struct ocf_operation oo;
127 int error;
128
129 oo.os = os;
130 oo.done = false;
131
132 crp->crp_opaque = &oo;
133 crp->crp_callback = ktls_ocf_callback;
134 for (;;) {
135 error = crypto_dispatch(crp);
136 if (error)
137 break;
138
139 mtx_lock(&os->lock);
140 while (!oo.done)
141 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
142 mtx_unlock(&os->lock);
143
144 if (crp->crp_etype != EAGAIN) {
145 error = crp->crp_etype;
146 break;
147 }
148
149 crp->crp_etype = 0;
150 crp->crp_flags &= ~CRYPTO_F_DONE;
151 oo.done = false;
152 counter_u64_add(ocf_retries, 1);
153 }
154 return (error);
155 }
156
157 static int
158 ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
159 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
160 struct iovec *outiov, int iovcnt, uint64_t seqno,
161 uint8_t record_type __unused)
162 {
163 struct uio uio, out_uio;
164 struct tls_mac_data ad;
165 struct cryptop crp;
166 struct ocf_session *os;
167 struct iovec iov[iovcnt + 2];
168 struct iovec out_iov[iovcnt + 1];
169 int i, error;
170 uint16_t tls_comp_len;
171 uint8_t pad;
172 bool inplace;
173
174 os = tls->cipher;
175
176 #ifdef INVARIANTS
177 if (os->implicit_iv) {
178 mtx_lock(&os->lock);
179 KASSERT(!os->in_progress,
180 ("concurrent implicit IV encryptions"));
181 if (os->next_seqno != seqno) {
182 printf("KTLS CBC: TLS records out of order. "
183 "Expected %ju, got %ju\n",
184 (uintmax_t)os->next_seqno, (uintmax_t)seqno);
185 mtx_unlock(&os->lock);
186 return (EINVAL);
187 }
188 os->in_progress = true;
189 mtx_unlock(&os->lock);
190 }
191 #endif
192
193 /*
194 * Compute the payload length.
195 *
196 * XXX: This could be easily computed O(1) from the mbuf
197 * fields, but we don't have those accessible here. Can
198 * at least compute inplace as well while we are here.
199 */
200 tls_comp_len = 0;
201 inplace = true;
202 for (i = 0; i < iovcnt; i++) {
203 tls_comp_len += iniov[i].iov_len;
204 if (iniov[i].iov_base != outiov[i].iov_base)
205 inplace = false;
206 }
207
208 /* Initialize the AAD. */
209 ad.seq = htobe64(seqno);
210 ad.type = hdr->tls_type;
211 ad.tls_vmajor = hdr->tls_vmajor;
212 ad.tls_vminor = hdr->tls_vminor;
213 ad.tls_length = htons(tls_comp_len);
214
215 /* First, compute the MAC. */
216 iov[0].iov_base = &ad;
217 iov[0].iov_len = sizeof(ad);
218 memcpy(&iov[1], iniov, sizeof(*iniov) * iovcnt);
219 iov[iovcnt + 1].iov_base = trailer;
220 iov[iovcnt + 1].iov_len = os->mac_len;
221 uio.uio_iov = iov;
222 uio.uio_iovcnt = iovcnt + 2;
223 uio.uio_offset = 0;
224 uio.uio_segflg = UIO_SYSSPACE;
225 uio.uio_td = curthread;
226 uio.uio_resid = sizeof(ad) + tls_comp_len + os->mac_len;
227
228 crypto_initreq(&crp, os->mac_sid);
229 crp.crp_payload_start = 0;
230 crp.crp_payload_length = sizeof(ad) + tls_comp_len;
231 crp.crp_digest_start = crp.crp_payload_length;
232 crp.crp_op = CRYPTO_OP_COMPUTE_DIGEST;
233 crp.crp_flags = CRYPTO_F_CBIMM;
234 crypto_use_uio(&crp, &uio);
235 error = ktls_ocf_dispatch(os, &crp);
236
237 crypto_destroyreq(&crp);
238 if (error) {
239 #ifdef INVARIANTS
240 if (os->implicit_iv) {
241 mtx_lock(&os->lock);
242 os->in_progress = false;
243 mtx_unlock(&os->lock);
244 }
245 #endif
246 return (error);
247 }
248
249 /* Second, add the padding. */
250 pad = (unsigned)(AES_BLOCK_LEN - (tls_comp_len + os->mac_len + 1)) %
251 AES_BLOCK_LEN;
252 for (i = 0; i < pad + 1; i++)
253 trailer[os->mac_len + i] = pad;
254
255 /* Finally, encrypt the record. */
256
257 /*
258 * Don't recopy the input iovec, instead just adjust the
259 * trailer length and skip over the AAD vector in the uio.
260 */
261 iov[iovcnt + 1].iov_len += pad + 1;
262 uio.uio_iov = iov + 1;
263 uio.uio_iovcnt = iovcnt + 1;
264 uio.uio_resid = tls_comp_len + iov[iovcnt + 1].iov_len;
265 KASSERT(uio.uio_resid % AES_BLOCK_LEN == 0,
266 ("invalid encryption size"));
267
268 crypto_initreq(&crp, os->sid);
269 crp.crp_payload_start = 0;
270 crp.crp_payload_length = uio.uio_resid;
271 crp.crp_op = CRYPTO_OP_ENCRYPT;
272 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
273 if (os->implicit_iv)
274 memcpy(crp.crp_iv, os->iv, AES_BLOCK_LEN);
275 else
276 memcpy(crp.crp_iv, hdr + 1, AES_BLOCK_LEN);
277 crypto_use_uio(&crp, &uio);
278 if (!inplace) {
279 memcpy(out_iov, outiov, sizeof(*iniov) * iovcnt);
280 out_iov[iovcnt] = iov[iovcnt + 1];
281 out_uio.uio_iov = out_iov;
282 out_uio.uio_iovcnt = iovcnt + 1;
283 out_uio.uio_offset = 0;
284 out_uio.uio_segflg = UIO_SYSSPACE;
285 out_uio.uio_td = curthread;
286 out_uio.uio_resid = uio.uio_resid;
287 crypto_use_output_uio(&crp, &out_uio);
288 }
289
290 if (os->implicit_iv)
291 counter_u64_add(ocf_tls10_cbc_crypts, 1);
292 else
293 counter_u64_add(ocf_tls11_cbc_crypts, 1);
294 if (inplace)
295 counter_u64_add(ocf_inplace, 1);
296 else
297 counter_u64_add(ocf_separate_output, 1);
298 error = ktls_ocf_dispatch(os, &crp);
299
300 crypto_destroyreq(&crp);
301
302 if (os->implicit_iv) {
303 KASSERT(os->mac_len + pad + 1 >= AES_BLOCK_LEN,
304 ("trailer too short to read IV"));
305 memcpy(os->iv, trailer + os->mac_len + pad + 1 - AES_BLOCK_LEN,
306 AES_BLOCK_LEN);
307 #ifdef INVARIANTS
308 mtx_lock(&os->lock);
309 os->next_seqno = seqno + 1;
310 os->in_progress = false;
311 mtx_unlock(&os->lock);
312 #endif
313 }
314 return (error);
315 }
316
317 static int
318 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
319 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
320 struct iovec *outiov, int iovcnt, uint64_t seqno,
321 uint8_t record_type __unused)
322 {
323 struct uio uio, out_uio, *tag_uio;
324 struct tls_aead_data ad;
325 struct cryptop crp;
326 struct ocf_session *os;
327 struct iovec iov[iovcnt + 1];
328 int i, error;
329 uint16_t tls_comp_len;
330 bool inplace;
331
332 os = tls->cipher;
333
334 uio.uio_iov = iniov;
335 uio.uio_iovcnt = iovcnt;
336 uio.uio_offset = 0;
337 uio.uio_segflg = UIO_SYSSPACE;
338 uio.uio_td = curthread;
339
340 out_uio.uio_iov = outiov;
341 out_uio.uio_iovcnt = iovcnt;
342 out_uio.uio_offset = 0;
343 out_uio.uio_segflg = UIO_SYSSPACE;
344 out_uio.uio_td = curthread;
345
346 crypto_initreq(&crp, os->sid);
347
348 /* Setup the IV. */
349 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
350 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
351
352 /* Setup the AAD. */
353 tls_comp_len = ntohs(hdr->tls_length) -
354 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
355 ad.seq = htobe64(seqno);
356 ad.type = hdr->tls_type;
357 ad.tls_vmajor = hdr->tls_vmajor;
358 ad.tls_vminor = hdr->tls_vminor;
359 ad.tls_length = htons(tls_comp_len);
360 crp.crp_aad = &ad;
361 crp.crp_aad_length = sizeof(ad);
362
363 /* Compute payload length and determine if encryption is in place. */
364 inplace = true;
365 crp.crp_payload_start = 0;
366 for (i = 0; i < iovcnt; i++) {
367 if (iniov[i].iov_base != outiov[i].iov_base)
368 inplace = false;
369 crp.crp_payload_length += iniov[i].iov_len;
370 }
371 uio.uio_resid = crp.crp_payload_length;
372 out_uio.uio_resid = crp.crp_payload_length;
373
374 if (inplace)
375 tag_uio = &uio;
376 else
377 tag_uio = &out_uio;
378
379 /* Duplicate iovec and append vector for tag. */
380 memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
381 iov[iovcnt].iov_base = trailer;
382 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN;
383 tag_uio->uio_iov = iov;
384 tag_uio->uio_iovcnt++;
385 crp.crp_digest_start = tag_uio->uio_resid;
386 tag_uio->uio_resid += AES_GMAC_HASH_LEN;
387
388 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
389 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
390 crypto_use_uio(&crp, &uio);
391 if (!inplace)
392 crypto_use_output_uio(&crp, &out_uio);
393
394 counter_u64_add(ocf_tls12_gcm_crypts, 1);
395 if (inplace)
396 counter_u64_add(ocf_inplace, 1);
397 else
398 counter_u64_add(ocf_separate_output, 1);
399 error = ktls_ocf_dispatch(os, &crp);
400
401 crypto_destroyreq(&crp);
402 return (error);
403 }
404
405 static int
406 ktls_ocf_tls12_gcm_decrypt(struct ktls_session *tls,
407 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
408 int *trailer_len)
409 {
410 struct tls_aead_data ad;
411 struct cryptop crp;
412 struct ocf_session *os;
413 struct ocf_operation oo;
414 int error;
415 uint16_t tls_comp_len;
416
417 os = tls->cipher;
418
419 oo.os = os;
420 oo.done = false;
421
422 crypto_initreq(&crp, os->sid);
423
424 /* Setup the IV. */
425 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
426 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
427
428 /* Setup the AAD. */
429 tls_comp_len = ntohs(hdr->tls_length) -
430 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
431 ad.seq = htobe64(seqno);
432 ad.type = hdr->tls_type;
433 ad.tls_vmajor = hdr->tls_vmajor;
434 ad.tls_vminor = hdr->tls_vminor;
435 ad.tls_length = htons(tls_comp_len);
436 crp.crp_aad = &ad;
437 crp.crp_aad_length = sizeof(ad);
438
439 crp.crp_payload_start = tls->params.tls_hlen;
440 crp.crp_payload_length = tls_comp_len;
441 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
442
443 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
444 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
445 crypto_use_mbuf(&crp, m);
446
447 counter_u64_add(ocf_tls12_gcm_crypts, 1);
448 error = ktls_ocf_dispatch(os, &crp);
449
450 crypto_destroyreq(&crp);
451 *trailer_len = AES_GMAC_HASH_LEN;
452 return (error);
453 }
454
455 static int
456 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
457 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
458 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
459 {
460 struct uio uio, out_uio;
461 struct tls_aead_data_13 ad;
462 char nonce[12];
463 struct cryptop crp;
464 struct ocf_session *os;
465 struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1];
466 int i, error;
467 bool inplace;
468
469 os = tls->cipher;
470
471 crypto_initreq(&crp, os->sid);
472
473 /* Setup the nonce. */
474 memcpy(nonce, tls->params.iv, tls->params.iv_len);
475 *(uint64_t *)(nonce + 4) ^= htobe64(seqno);
476
477 /* Setup the AAD. */
478 ad.type = hdr->tls_type;
479 ad.tls_vmajor = hdr->tls_vmajor;
480 ad.tls_vminor = hdr->tls_vminor;
481 ad.tls_length = hdr->tls_length;
482 crp.crp_aad = &ad;
483 crp.crp_aad_length = sizeof(ad);
484
485 /* Compute payload length and determine if encryption is in place. */
486 inplace = true;
487 crp.crp_payload_start = 0;
488 for (i = 0; i < iovcnt; i++) {
489 if (iniov[i].iov_base != outiov[i].iov_base)
490 inplace = false;
491 crp.crp_payload_length += iniov[i].iov_len;
492 }
493
494 /* Store the record type as the first byte of the trailer. */
495 trailer[0] = record_type;
496 crp.crp_payload_length++;
497 crp.crp_digest_start = crp.crp_payload_length;
498
499 /*
500 * Duplicate the input iov to append the trailer. Always
501 * include the full trailer as input to get the record_type
502 * even if only the first byte is used.
503 */
504 memcpy(iov, iniov, iovcnt * sizeof(*iov));
505 iov[iovcnt].iov_base = trailer;
506 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1;
507 uio.uio_iov = iov;
508 uio.uio_iovcnt = iovcnt + 1;
509 uio.uio_offset = 0;
510 uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN;
511 uio.uio_segflg = UIO_SYSSPACE;
512 uio.uio_td = curthread;
513 crypto_use_uio(&crp, &uio);
514
515 if (!inplace) {
516 /* Duplicate the output iov to append the trailer. */
517 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
518 out_iov[iovcnt] = iov[iovcnt];
519
520 out_uio.uio_iov = out_iov;
521 out_uio.uio_iovcnt = iovcnt + 1;
522 out_uio.uio_offset = 0;
523 out_uio.uio_resid = crp.crp_payload_length +
524 AES_GMAC_HASH_LEN;
525 out_uio.uio_segflg = UIO_SYSSPACE;
526 out_uio.uio_td = curthread;
527 crypto_use_output_uio(&crp, &out_uio);
528 }
529
530 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
531 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
532
533 memcpy(crp.crp_iv, nonce, sizeof(nonce));
534
535 counter_u64_add(ocf_tls13_gcm_crypts, 1);
536 if (inplace)
537 counter_u64_add(ocf_inplace, 1);
538 else
539 counter_u64_add(ocf_separate_output, 1);
540 error = ktls_ocf_dispatch(os, &crp);
541
542 crypto_destroyreq(&crp);
543 return (error);
544 }
545
546 static void
547 ktls_ocf_free(struct ktls_session *tls)
548 {
549 struct ocf_session *os;
550
551 os = tls->cipher;
552 crypto_freesession(os->sid);
553 mtx_destroy(&os->lock);
554 zfree(os, M_KTLS_OCF);
555 }
556
557 static int
558 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
559 {
560 struct crypto_session_params csp, mac_csp;
561 struct ocf_session *os;
562 int error, mac_len;
563
564 memset(&csp, 0, sizeof(csp));
565 memset(&mac_csp, 0, sizeof(mac_csp));
566 mac_csp.csp_mode = CSP_MODE_NONE;
567 mac_len = 0;
568
569 switch (tls->params.cipher_algorithm) {
570 case CRYPTO_AES_NIST_GCM_16:
571 switch (tls->params.cipher_key_len) {
572 case 128 / 8:
573 case 256 / 8:
574 break;
575 default:
576 return (EINVAL);
577 }
578
579 /* Only TLS 1.2 and 1.3 are supported. */
580 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
581 tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
582 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
583 return (EPROTONOSUPPORT);
584
585 /* TLS 1.3 is not yet supported for receive. */
586 if (direction == KTLS_RX &&
587 tls->params.tls_vminor == TLS_MINOR_VER_THREE)
588 return (EPROTONOSUPPORT);
589
590 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
591 csp.csp_mode = CSP_MODE_AEAD;
592 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
593 csp.csp_cipher_key = tls->params.cipher_key;
594 csp.csp_cipher_klen = tls->params.cipher_key_len;
595 csp.csp_ivlen = AES_GCM_IV_LEN;
596 break;
597 case CRYPTO_AES_CBC:
598 switch (tls->params.cipher_key_len) {
599 case 128 / 8:
600 case 256 / 8:
601 break;
602 default:
603 return (EINVAL);
604 }
605
606 switch (tls->params.auth_algorithm) {
607 case CRYPTO_SHA1_HMAC:
608 mac_len = SHA1_HASH_LEN;
609 break;
610 case CRYPTO_SHA2_256_HMAC:
611 mac_len = SHA2_256_HASH_LEN;
612 break;
613 case CRYPTO_SHA2_384_HMAC:
614 mac_len = SHA2_384_HASH_LEN;
615 break;
616 default:
617 return (EINVAL);
618 }
619
620 /* Only TLS 1.0-1.2 are supported. */
621 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
622 tls->params.tls_vminor < TLS_MINOR_VER_ZERO ||
623 tls->params.tls_vminor > TLS_MINOR_VER_TWO)
624 return (EPROTONOSUPPORT);
625
626 /* AES-CBC is not supported for receive. */
627 if (direction == KTLS_RX)
628 return (EPROTONOSUPPORT);
629
630 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
631 csp.csp_mode = CSP_MODE_CIPHER;
632 csp.csp_cipher_alg = CRYPTO_AES_CBC;
633 csp.csp_cipher_key = tls->params.cipher_key;
634 csp.csp_cipher_klen = tls->params.cipher_key_len;
635 csp.csp_ivlen = AES_BLOCK_LEN;
636
637 mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
638 mac_csp.csp_mode = CSP_MODE_DIGEST;
639 mac_csp.csp_auth_alg = tls->params.auth_algorithm;
640 mac_csp.csp_auth_key = tls->params.auth_key;
641 mac_csp.csp_auth_klen = tls->params.auth_key_len;
642 break;
643 default:
644 return (EPROTONOSUPPORT);
645 }
646
647 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
648 if (os == NULL)
649 return (ENOMEM);
650
651 error = crypto_newsession(&os->sid, &csp,
652 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
653 if (error) {
654 free(os, M_KTLS_OCF);
655 return (error);
656 }
657
658 if (mac_csp.csp_mode != CSP_MODE_NONE) {
659 error = crypto_newsession(&os->mac_sid, &mac_csp,
660 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
661 if (error) {
662 crypto_freesession(os->sid);
663 free(os, M_KTLS_OCF);
664 return (error);
665 }
666 os->mac_len = mac_len;
667 }
668
669 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
670 tls->cipher = os;
671 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
672 if (direction == KTLS_TX) {
673 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
674 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
675 else
676 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
677 } else {
678 tls->sw_decrypt = ktls_ocf_tls12_gcm_decrypt;
679 }
680 } else {
681 tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;
682 if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {
683 os->implicit_iv = true;
684 memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);
685 }
686 }
687 tls->free = ktls_ocf_free;
688 return (0);
689 }
690
691 struct ktls_crypto_backend ocf_backend = {
692 .name = "OCF",
693 .prio = 5,
694 .api_version = KTLS_API_VERSION,
695 .try = ktls_ocf_try,
696 };
697
698 static int
699 ktls_ocf_modevent(module_t mod, int what, void *arg)
700 {
701 int error;
702
703 switch (what) {
704 case MOD_LOAD:
705 ocf_tls10_cbc_crypts = counter_u64_alloc(M_WAITOK);
706 ocf_tls11_cbc_crypts = counter_u64_alloc(M_WAITOK);
707 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
708 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
709 ocf_inplace = counter_u64_alloc(M_WAITOK);
710 ocf_separate_output = counter_u64_alloc(M_WAITOK);
711 ocf_retries = counter_u64_alloc(M_WAITOK);
712 return (ktls_crypto_backend_register(&ocf_backend));
713 case MOD_UNLOAD:
714 error = ktls_crypto_backend_deregister(&ocf_backend);
715 if (error)
716 return (error);
717 counter_u64_free(ocf_tls10_cbc_crypts);
718 counter_u64_free(ocf_tls11_cbc_crypts);
719 counter_u64_free(ocf_tls12_gcm_crypts);
720 counter_u64_free(ocf_tls13_gcm_crypts);
721 counter_u64_free(ocf_inplace);
722 counter_u64_free(ocf_separate_output);
723 counter_u64_free(ocf_retries);
724 return (0);
725 default:
726 return (EOPNOTSUPP);
727 }
728 }
729
730 static moduledata_t ktls_ocf_moduledata = {
731 "ktls_ocf",
732 ktls_ocf_modevent,
733 NULL
734 };
735
736 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);
Cache object: 6d054f6a2bc912a96585526b0e0fe495
|