FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_msan.c
1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 * Copyright (c) 2021 The FreeBSD Foundation
7 *
8 * Portions of this software were developed by Mark Johnston under sponsorship
9 * from the FreeBSD Foundation.
10 *
11 * This code is part of the KMSAN subsystem of the NetBSD kernel.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #define SAN_RUNTIME
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 #if 0
40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
41 #endif
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/bio.h>
46 #include <sys/buf.h>
47 #include <sys/conf.h>
48 #include <sys/kdb.h>
49 #include <sys/kernel.h>
50 #include <sys/linker.h>
51 #include <sys/malloc.h>
52 #include <sys/mbuf.h>
53 #include <sys/memdesc.h>
54 #include <sys/msan.h>
55 #include <sys/proc.h>
56 #include <sys/stack.h>
57 #include <sys/sysctl.h>
58 #include <sys/uio.h>
59
60 #include <cam/cam.h>
61 #include <cam/cam_ccb.h>
62
63 #include <vm/vm.h>
64 #include <vm/pmap.h>
65
66 #include <machine/msan.h>
67 #include <machine/stdarg.h>
68
69 void kmsan_init_arg(size_t);
70 void kmsan_init_ret(size_t);
71
72 /* -------------------------------------------------------------------------- */
73
74 /*
75 * Part of the compiler ABI.
76 */
77
78 typedef struct {
79 uint8_t *shad;
80 msan_orig_t *orig;
81 } msan_meta_t;
82
83 #define MSAN_PARAM_SIZE 800
84 #define MSAN_RETVAL_SIZE 800
85 typedef struct {
86 uint8_t param_shadow[MSAN_PARAM_SIZE];
87 uint8_t retval_shadow[MSAN_RETVAL_SIZE];
88 uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
89 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
90 uint64_t va_arg_overflow_size;
91 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
92 msan_orig_t retval_origin;
93 } msan_tls_t;
94
95 /* -------------------------------------------------------------------------- */
96
97 #define MSAN_NCONTEXT 4
98 #define MSAN_ORIG_MASK (~0x3)
99
100 typedef struct kmsan_td {
101 size_t ctx;
102 msan_tls_t tls[MSAN_NCONTEXT];
103 } msan_td_t;
104
105 static msan_tls_t dummy_tls;
106
107 /*
108 * Use separate dummy regions for loads and stores: stores may mark the region
109 * as uninitialized, and that can trigger false positives.
110 */
111 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
112 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
113 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
114 static msan_td_t msan_thread0;
115 static bool kmsan_enabled __read_mostly;
116
117 static bool kmsan_reporting = false;
118
119 /*
120 * Avoid clobbering any thread-local state before we panic.
121 */
122 #define kmsan_panic(f, ...) do { \
123 kmsan_enabled = false; \
124 panic(f, __VA_ARGS__); \
125 } while (0)
126
127 #define REPORT(f, ...) do { \
128 if (panic_on_violation) { \
129 kmsan_panic(f, __VA_ARGS__); \
130 } else { \
131 struct stack st; \
132 \
133 stack_save(&st); \
134 printf(f "\n", __VA_ARGS__); \
135 stack_print_ddb(&st); \
136 } \
137 } while (0)
138
139 FEATURE(kmsan, "Kernel memory sanitizer");
140
141 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
142 "KMSAN options");
143
144 static bool panic_on_violation = 1;
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
146 &panic_on_violation, 0,
147 "Panic if an invalid access is detected");
148
149 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
150
151 /* -------------------------------------------------------------------------- */
152
153 static inline const char *
154 kmsan_orig_name(int type)
155 {
156 switch (type) {
157 case KMSAN_TYPE_STACK:
158 return ("stack");
159 case KMSAN_TYPE_KMEM:
160 return ("kmem");
161 case KMSAN_TYPE_MALLOC:
162 return ("malloc");
163 case KMSAN_TYPE_UMA:
164 return ("UMA");
165 default:
166 return ("unknown");
167 }
168 }
169
170 static void
171 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
172 {
173 msan_orig_t *orig;
174 const char *typename;
175 char *var, *fn;
176 uintptr_t ptr;
177 long foff;
178 char buf[128];
179 int type;
180
181 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
182 return;
183
184 kmsan_reporting = true;
185 __compiler_membar();
186
187 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
188 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
189
190 if (*orig == 0) {
191 REPORT("MSan: Uninitialized memory in %s, offset %zu",
192 hook, off);
193 goto out;
194 }
195
196 kmsan_md_orig_decode(*orig, &type, &ptr);
197 typename = kmsan_orig_name(type);
198
199 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
200 sizeof(buf), &foff) == 0) {
201 REPORT("MSan: Uninitialized %s memory in %s, "
202 "offset %zu/%zu, addr %p, from %s+%#lx",
203 typename, hook, off, size, addr, buf, foff);
204 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
205 /*
206 * The format of the string is: "----var@function". Parse it to
207 * display a nice warning.
208 */
209 var = (char *)ptr + 4;
210 strlcpy(buf, var, sizeof(buf));
211 var = buf;
212 fn = strchr(buf, '@');
213 *fn++ = '\0';
214 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
215 "variable '%s' from %s", typename, hook, off, var, fn);
216 } else {
217 REPORT("MSan: Uninitialized %s memory in %s, "
218 "offset %zu/%zu, addr %p, PC %p",
219 typename, hook, off, size, addr, (void *)ptr);
220 }
221
222 out:
223 __compiler_membar();
224 kmsan_reporting = false;
225 }
226
227 static void
228 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
229 {
230 const char *typename;
231 char *var, *fn;
232 uintptr_t ptr;
233 char buf[128];
234 long foff;
235 int type;
236
237 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
238 return;
239
240 kmsan_reporting = true;
241 __compiler_membar();
242
243 if (orig == 0) {
244 REPORT("MSan: uninitialized variable in %p", (void *)pc);
245 goto out;
246 }
247
248 kmsan_md_orig_decode(orig, &type, &ptr);
249 typename = kmsan_orig_name(type);
250
251 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
252 sizeof(buf), &foff) == 0) {
253 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
254 typename, buf, foff);
255 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
256 /*
257 * The format of the string is: "----var@function". Parse it to
258 * display a nice warning.
259 */
260 var = (char *)ptr + 4;
261 strlcpy(buf, var, sizeof(buf));
262 var = buf;
263 fn = strchr(buf, '@');
264 *fn++ = '\0';
265 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
266 } else {
267 REPORT("MSan: Uninitialized %s memory, origin %x",
268 typename, orig);
269 }
270
271 out:
272 __compiler_membar();
273 kmsan_reporting = false;
274 }
275
276 /* -------------------------------------------------------------------------- */
277
278 static inline msan_meta_t
279 kmsan_meta_get(const void *addr, size_t size, const bool write)
280 {
281 msan_meta_t ret;
282
283 if (__predict_false(!kmsan_enabled)) {
284 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
285 ret.orig = (msan_orig_t *)msan_dummy_orig;
286 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
287 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
288 ret.orig = (msan_orig_t *)msan_dummy_orig;
289 } else {
290 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
291 ret.orig =
292 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
293 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
294 MSAN_ORIG_MASK);
295 }
296
297 return (ret);
298 }
299
300 static inline void
301 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
302 {
303 msan_orig_t *orig;
304 size_t i;
305
306 if (__predict_false(!kmsan_enabled))
307 return;
308 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
309 return;
310
311 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
312 size += ((uintptr_t)orig & (sizeof(*orig) - 1));
313 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
314
315 for (i = 0; i < size; i += 4) {
316 orig[i / 4] = o;
317 }
318 }
319
320 static inline void
321 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
322 {
323 uint8_t *shad;
324
325 if (__predict_false(!kmsan_enabled))
326 return;
327 if (__predict_false(kmsan_md_unsupported(addr)))
328 return;
329
330 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
331 __builtin_memset(shad, c, size);
332 }
333
334 static inline void
335 kmsan_meta_copy(void *dst, const void *src, size_t size)
336 {
337 uint8_t *orig_src, *orig_dst;
338 uint8_t *shad_src, *shad_dst;
339 msan_orig_t *_src, *_dst;
340 size_t i;
341
342 if (__predict_false(!kmsan_enabled))
343 return;
344 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
345 return;
346 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
347 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
348 return;
349 }
350
351 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
352 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
353 __builtin_memmove(shad_dst, shad_src, size);
354
355 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
356 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
357 for (i = 0; i < size; i++) {
358 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
359 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
360 *_dst = *_src;
361 orig_src++;
362 orig_dst++;
363 }
364 }
365
366 static inline void
367 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
368 {
369 uint8_t *shad;
370 size_t i;
371
372 if (__predict_false(!kmsan_enabled))
373 return;
374 if (__predict_false(kmsan_md_unsupported(addr)))
375 return;
376
377 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
378 for (i = 0; i < size; i++) {
379 if (__predict_true(shad[i] == 0))
380 continue;
381 kmsan_report_hook((const char *)addr + i, size, i, hook);
382 break;
383 }
384 }
385
386 void
387 kmsan_init_arg(size_t n)
388 {
389 msan_td_t *mtd;
390 uint8_t *arg;
391
392 if (__predict_false(!kmsan_enabled))
393 return;
394 if (__predict_false(curthread == NULL))
395 return;
396 mtd = curthread->td_kmsan;
397 arg = mtd->tls[mtd->ctx].param_shadow;
398 __builtin_memset(arg, 0, n);
399 }
400
401 void
402 kmsan_init_ret(size_t n)
403 {
404 msan_td_t *mtd;
405 uint8_t *arg;
406
407 if (__predict_false(!kmsan_enabled))
408 return;
409 if (__predict_false(curthread == NULL))
410 return;
411 mtd = curthread->td_kmsan;
412 arg = mtd->tls[mtd->ctx].retval_shadow;
413 __builtin_memset(arg, 0, n);
414 }
415
416 static void
417 kmsan_check_arg(size_t size, const char *hook)
418 {
419 msan_td_t *mtd;
420 uint8_t *arg;
421 size_t i;
422
423 if (__predict_false(!kmsan_enabled))
424 return;
425 if (__predict_false(curthread == NULL))
426 return;
427 mtd = curthread->td_kmsan;
428 arg = mtd->tls[mtd->ctx].param_shadow;
429
430 for (i = 0; i < size; i++) {
431 if (__predict_true(arg[i] == 0))
432 continue;
433 kmsan_report_hook((const char *)arg + i, size, i, hook);
434 break;
435 }
436 }
437
438 void
439 kmsan_thread_alloc(struct thread *td)
440 {
441 msan_td_t *mtd;
442
443 if (!kmsan_enabled)
444 return;
445
446 mtd = td->td_kmsan;
447 if (mtd == NULL) {
448 /* We might be recycling a thread. */
449 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
450 sizeof(int));
451 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
452 }
453 kmsan_memset(mtd, 0, sizeof(*mtd));
454 mtd->ctx = 0;
455
456 if (td->td_kstack != 0)
457 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
458 KMSAN_STATE_UNINIT);
459
460 td->td_kmsan = mtd;
461 }
462
463 void
464 kmsan_thread_free(struct thread *td)
465 {
466 msan_td_t *mtd;
467
468 if (!kmsan_enabled)
469 return;
470 if (__predict_false(td == curthread))
471 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
472
473 mtd = td->td_kmsan;
474 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
475 free(mtd, M_KMSAN);
476 td->td_kmsan = NULL;
477 }
478
479 void kmsan_intr_enter(void);
480 void kmsan_intr_leave(void);
481
482 void
483 kmsan_intr_enter(void)
484 {
485 msan_td_t *mtd;
486
487 if (__predict_false(!kmsan_enabled))
488 return;
489
490 mtd = curthread->td_kmsan;
491 mtd->ctx++;
492 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
493 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
494 }
495
496 void
497 kmsan_intr_leave(void)
498 {
499 msan_td_t *mtd;
500
501 if (__predict_false(!kmsan_enabled))
502 return;
503
504 mtd = curthread->td_kmsan;
505 if (__predict_false(mtd->ctx == 0))
506 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
507 mtd->ctx--;
508 }
509
510 /* -------------------------------------------------------------------------- */
511
512 void
513 kmsan_shadow_map(vm_offset_t addr, size_t size)
514 {
515 size_t npages, i;
516 vm_offset_t va;
517
518 MPASS(addr % PAGE_SIZE == 0);
519 MPASS(size % PAGE_SIZE == 0);
520
521 if (!kmsan_enabled)
522 return;
523
524 npages = atop(size);
525
526 va = kmsan_md_addr_to_shad(addr);
527 for (i = 0; i < npages; i++) {
528 pmap_san_enter(va + ptoa(i));
529 }
530
531 va = kmsan_md_addr_to_orig(addr);
532 for (i = 0; i < npages; i++) {
533 pmap_san_enter(va + ptoa(i));
534 }
535 }
536
537 void
538 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
539 {
540 msan_orig_t orig;
541
542 orig = kmsan_md_orig_encode(type, pc);
543 kmsan_origin_fill(addr, orig, size);
544 }
545
546 void
547 kmsan_mark(const void *addr, size_t size, uint8_t c)
548 {
549 kmsan_shadow_fill((uintptr_t)addr, c, size);
550 }
551
552 void
553 kmsan_mark_bio(const struct bio *bp, uint8_t c)
554 {
555 kmsan_mark(bp->bio_data, bp->bio_length, c);
556 }
557
558 static void
559 kmsan_mark_ccb(const union ccb *ccb, uint8_t c)
560 {
561 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN)
562 return;
563 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
564 return;
565
566 switch (ccb->ccb_h.func_code) {
567 case XPT_SCSI_IO: {
568 const struct ccb_scsiio *scsiio;
569
570 scsiio = &ccb->ctio;
571 kmsan_mark(scsiio->data_ptr, scsiio->dxfer_len, c);
572 break;
573 }
574 case XPT_ATA_IO: {
575 const struct ccb_ataio *ataio;
576
577 ataio = &ccb->ataio;
578 kmsan_mark(ataio->data_ptr, ataio->dxfer_len, c);
579 break;
580 }
581 case XPT_NVME_IO: {
582 const struct ccb_nvmeio *nvmeio;
583
584 nvmeio = &ccb->nvmeio;
585 kmsan_mark(nvmeio->data_ptr, nvmeio->dxfer_len, c);
586 break;
587 }
588 default:
589 kmsan_panic("%s: unhandled CCB type %d", __func__,
590 ccb->ccb_h.func_code);
591 }
592 }
593
594 void
595 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
596 {
597 do {
598 if ((m->m_flags & M_EXTPG) == 0)
599 kmsan_mark(m->m_data, m->m_len, c);
600 m = m->m_next;
601 } while (m != NULL);
602 }
603
604 void
605 kmsan_check(const void *p, size_t sz, const char *descr)
606 {
607 kmsan_shadow_check((uintptr_t)p, sz, descr);
608 }
609
610 void
611 kmsan_check_bio(const struct bio *bp, const char *descr)
612 {
613 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
614 }
615
616 void
617 kmsan_check_ccb(const union ccb *ccb, const char *descr)
618 {
619 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT)
620 return;
621 switch (ccb->ccb_h.func_code) {
622 case XPT_SCSI_IO: {
623 const struct ccb_scsiio *scsiio;
624
625 scsiio = &ccb->ctio;
626 kmsan_check(scsiio->data_ptr, scsiio->dxfer_len, descr);
627 break;
628 }
629 case XPT_ATA_IO: {
630 const struct ccb_ataio *ataio;
631
632 ataio = &ccb->ataio;
633 kmsan_check(ataio->data_ptr, ataio->dxfer_len, descr);
634 break;
635 }
636 case XPT_NVME_IO: {
637 const struct ccb_nvmeio *nvmeio;
638
639 nvmeio = &ccb->nvmeio;
640 kmsan_check(nvmeio->data_ptr, nvmeio->dxfer_len, descr);
641 break;
642 }
643 default:
644 kmsan_panic("%s: unhandled CCB type %d", __func__,
645 ccb->ccb_h.func_code);
646 }
647 }
648
649 void
650 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
651 {
652 do {
653 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
654 } while ((m = m->m_next) != NULL);
655 }
656
657 void
658 kmsan_init(void)
659 {
660 int disabled;
661
662 disabled = 0;
663 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
664 if (disabled)
665 return;
666
667 /* Initialize the TLS for curthread. */
668 msan_thread0.ctx = 0;
669 thread0.td_kmsan = &msan_thread0;
670
671 /* Now officially enabled. */
672 kmsan_enabled = true;
673 }
674
675 /* -------------------------------------------------------------------------- */
676
677 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
678 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
679
680 msan_meta_t
681 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
682 {
683 return (kmsan_meta_get(addr, size, false));
684 }
685
686 msan_meta_t
687 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
688 {
689 return (kmsan_meta_get(addr, size, true));
690 }
691
692 #define MSAN_META_FUNC(size) \
693 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
694 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
695 { \
696 return (kmsan_meta_get(addr, size, false)); \
697 } \
698 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
699 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
700 { \
701 return (kmsan_meta_get(addr, size, true)); \
702 }
703
704 MSAN_META_FUNC(1)
705 MSAN_META_FUNC(2)
706 MSAN_META_FUNC(4)
707 MSAN_META_FUNC(8)
708
709 void __msan_instrument_asm_store(const void *, size_t);
710 msan_orig_t __msan_chain_origin(msan_orig_t);
711 void __msan_poison(const void *, size_t);
712 void __msan_unpoison(const void *, size_t);
713 void __msan_poison_alloca(const void *, uint64_t, const char *);
714 void __msan_unpoison_alloca(const void *, uint64_t);
715 void __msan_warning(msan_orig_t);
716 msan_tls_t *__msan_get_context_state(void);
717
718 void
719 __msan_instrument_asm_store(const void *addr, size_t size)
720 {
721 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
722 }
723
724 msan_orig_t
725 __msan_chain_origin(msan_orig_t origin)
726 {
727 return (origin);
728 }
729
730 void
731 __msan_poison(const void *addr, size_t size)
732 {
733 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
734 }
735
736 void
737 __msan_unpoison(const void *addr, size_t size)
738 {
739 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
740 }
741
742 void
743 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
744 {
745 msan_orig_t orig;
746
747 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
748 kmsan_origin_fill(addr, orig, size);
749 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
750 }
751
752 void
753 __msan_unpoison_alloca(const void *addr, uint64_t size)
754 {
755 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
756 }
757
758 void
759 __msan_warning(msan_orig_t origin)
760 {
761 if (__predict_false(!kmsan_enabled))
762 return;
763 kmsan_report_inline(origin, KMSAN_RET_ADDR);
764 }
765
766 msan_tls_t *
767 __msan_get_context_state(void)
768 {
769 msan_td_t *mtd;
770
771 /*
772 * When APs are started, they execute some C code before curthread is
773 * set. We have to handle that here.
774 */
775 if (__predict_false(!kmsan_enabled || curthread == NULL))
776 return (&dummy_tls);
777 mtd = curthread->td_kmsan;
778 return (&mtd->tls[mtd->ctx]);
779 }
780
781 /* -------------------------------------------------------------------------- */
782
783 /*
784 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
785 * initialized areas properly.
786 */
787
788 void *
789 kmsan_memcpy(void *dst, const void *src, size_t len)
790 {
791 /* No kmsan_check_arg, because inlined. */
792 kmsan_init_ret(sizeof(void *));
793 if (__predict_true(len != 0)) {
794 kmsan_meta_copy(dst, src, len);
795 }
796 return (__builtin_memcpy(dst, src, len));
797 }
798
799 int
800 kmsan_memcmp(const void *b1, const void *b2, size_t len)
801 {
802 const uint8_t *_b1 = b1, *_b2 = b2;
803 size_t i;
804
805 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
806 "memcmp():args");
807 kmsan_init_ret(sizeof(int));
808
809 for (i = 0; i < len; i++) {
810 if (*_b1 != *_b2) {
811 kmsan_shadow_check((uintptr_t)b1, i + 1,
812 "memcmp():arg1");
813 kmsan_shadow_check((uintptr_t)b2, i + 1,
814 "memcmp():arg2");
815 return (*_b1 - *_b2);
816 }
817 _b1++, _b2++;
818 }
819
820 return (0);
821 }
822
823 void *
824 kmsan_memset(void *dst, int c, size_t len)
825 {
826 /* No kmsan_check_arg, because inlined. */
827 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
828 kmsan_init_ret(sizeof(void *));
829 return (__builtin_memset(dst, c, len));
830 }
831
832 void *
833 kmsan_memmove(void *dst, const void *src, size_t len)
834 {
835 /* No kmsan_check_arg, because inlined. */
836 if (__predict_true(len != 0)) {
837 kmsan_meta_copy(dst, src, len);
838 }
839 kmsan_init_ret(sizeof(void *));
840 return (__builtin_memmove(dst, src, len));
841 }
842
843 __strong_reference(kmsan_memcpy, __msan_memcpy);
844 __strong_reference(kmsan_memset, __msan_memset);
845 __strong_reference(kmsan_memmove, __msan_memmove);
846
847 char *
848 kmsan_strcpy(char *dst, const char *src)
849 {
850 const char *_src = src;
851 char *_dst = dst;
852 size_t len = 0;
853
854 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
855
856 while (1) {
857 len++;
858 *dst = *src;
859 if (*src == '\0')
860 break;
861 src++, dst++;
862 }
863
864 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
865 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
866 kmsan_init_ret(sizeof(char *));
867 return (_dst);
868 }
869
870 int
871 kmsan_strcmp(const char *s1, const char *s2)
872 {
873 const char *_s1 = s1, *_s2 = s2;
874 size_t len = 0;
875
876 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
877 kmsan_init_ret(sizeof(int));
878
879 while (1) {
880 len++;
881 if (*s1 != *s2)
882 break;
883 if (*s1 == '\0') {
884 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
885 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
886 return (0);
887 }
888 s1++, s2++;
889 }
890
891 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
892 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
893
894 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
895 }
896
897 size_t
898 kmsan_strlen(const char *str)
899 {
900 const char *s;
901
902 kmsan_check_arg(sizeof(str), "strlen():args");
903
904 s = str;
905 while (1) {
906 if (*s == '\0')
907 break;
908 s++;
909 }
910
911 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
912 kmsan_init_ret(sizeof(size_t));
913 return (s - str);
914 }
915
916 int kmsan_copyin(const void *, void *, size_t);
917 int kmsan_copyout(const void *, void *, size_t);
918 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
919
920 int
921 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
922 {
923 int ret;
924
925 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
926 "copyin():args");
927 ret = copyin(uaddr, kaddr, len);
928 if (ret == 0)
929 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
930 kmsan_init_ret(sizeof(int));
931 return (ret);
932 }
933
934 int
935 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
936 {
937 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
938 "copyout():args");
939 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
940 kmsan_init_ret(sizeof(int));
941 return (copyout(kaddr, uaddr, len));
942 }
943
944 int
945 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
946 {
947 size_t _done;
948 int ret;
949
950 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
951 sizeof(len) + sizeof(done), "copyinstr():args");
952 ret = copyinstr(uaddr, kaddr, len, &_done);
953 if (ret == 0)
954 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
955 if (done != NULL) {
956 *done = _done;
957 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
958 }
959 kmsan_init_ret(sizeof(int));
960 return (ret);
961 }
962
963 /* -------------------------------------------------------------------------- */
964
965 int
966 kmsan_fubyte(volatile const void *base)
967 {
968 int ret;
969
970 kmsan_check_arg(sizeof(base), "fubyte(): args");
971 ret = fubyte(base);
972 kmsan_init_ret(sizeof(int));
973 return (ret);
974 }
975
976 int
977 kmsan_fuword16(volatile const void *base)
978 {
979 int ret;
980
981 kmsan_check_arg(sizeof(base), "fuword16(): args");
982 ret = fuword16(base);
983 kmsan_init_ret(sizeof(int));
984 return (ret);
985 }
986
987 int
988 kmsan_fueword(volatile const void *base, long *val)
989 {
990 int ret;
991
992 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
993 ret = fueword(base, val);
994 if (ret == 0)
995 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
996 sizeof(*val));
997 kmsan_init_ret(sizeof(int));
998 return (ret);
999 }
1000
1001 int
1002 kmsan_fueword32(volatile const void *base, int32_t *val)
1003 {
1004 int ret;
1005
1006 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
1007 ret = fueword32(base, val);
1008 if (ret == 0)
1009 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
1010 sizeof(*val));
1011 kmsan_init_ret(sizeof(int));
1012 return (ret);
1013 }
1014
1015 int
1016 kmsan_fueword64(volatile const void *base, int64_t *val)
1017 {
1018 int ret;
1019
1020 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
1021 ret = fueword64(base, val);
1022 if (ret == 0)
1023 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
1024 sizeof(*val));
1025 kmsan_init_ret(sizeof(int));
1026 return (ret);
1027 }
1028
1029 int
1030 kmsan_subyte(volatile void *base, int byte)
1031 {
1032 int ret;
1033
1034 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
1035 ret = subyte(base, byte);
1036 kmsan_init_ret(sizeof(int));
1037 return (ret);
1038 }
1039
1040 int
1041 kmsan_suword(volatile void *base, long word)
1042 {
1043 int ret;
1044
1045 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
1046 ret = suword(base, word);
1047 kmsan_init_ret(sizeof(int));
1048 return (ret);
1049 }
1050
1051 int
1052 kmsan_suword16(volatile void *base, int word)
1053 {
1054 int ret;
1055
1056 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
1057 ret = suword16(base, word);
1058 kmsan_init_ret(sizeof(int));
1059 return (ret);
1060 }
1061
1062 int
1063 kmsan_suword32(volatile void *base, int32_t word)
1064 {
1065 int ret;
1066
1067 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1068 ret = suword32(base, word);
1069 kmsan_init_ret(sizeof(int));
1070 return (ret);
1071 }
1072
1073 int
1074 kmsan_suword64(volatile void *base, int64_t word)
1075 {
1076 int ret;
1077
1078 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1079 ret = suword64(base, word);
1080 kmsan_init_ret(sizeof(int));
1081 return (ret);
1082 }
1083
1084 int
1085 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1086 uint32_t newval)
1087 {
1088 int ret;
1089
1090 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1091 sizeof(newval), "casueword32(): args");
1092 ret = casueword32(base, oldval, oldvalp, newval);
1093 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1094 sizeof(*oldvalp));
1095 kmsan_init_ret(sizeof(int));
1096 return (ret);
1097 }
1098
1099 int
1100 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1101 u_long newval)
1102 {
1103 int ret;
1104
1105 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1106 sizeof(newval), "casueword32(): args");
1107 ret = casueword(base, oldval, oldvalp, newval);
1108 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1109 sizeof(*oldvalp));
1110 kmsan_init_ret(sizeof(int));
1111 return (ret);
1112 }
1113
1114 /* -------------------------------------------------------------------------- */
1115
1116 #include <machine/atomic.h>
1117 #include <sys/atomic_san.h>
1118
1119 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \
1120 void kmsan_atomic_add_##name(volatile type *ptr, type val) \
1121 { \
1122 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1123 "atomic_add_" #name "():args"); \
1124 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1125 "atomic_add_" #name "():ptr"); \
1126 atomic_add_##name(ptr, val); \
1127 }
1128
1129 #define MSAN_ATOMIC_FUNC_ADD(name, type) \
1130 _MSAN_ATOMIC_FUNC_ADD(name, type) \
1131 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
1132 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1133
1134 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1135 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1136 { \
1137 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1138 "atomic_subtract_" #name "():args"); \
1139 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1140 "atomic_subtract_" #name "():ptr"); \
1141 atomic_subtract_##name(ptr, val); \
1142 }
1143
1144 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1145 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1146 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
1147 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1148
1149 #define _MSAN_ATOMIC_FUNC_SET(name, type) \
1150 void kmsan_atomic_set_##name(volatile type *ptr, type val) \
1151 { \
1152 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1153 "atomic_set_" #name "():args"); \
1154 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1155 "atomic_set_" #name "():ptr"); \
1156 atomic_set_##name(ptr, val); \
1157 }
1158
1159 #define MSAN_ATOMIC_FUNC_SET(name, type) \
1160 _MSAN_ATOMIC_FUNC_SET(name, type) \
1161 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \
1162 _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1163
1164 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1165 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \
1166 { \
1167 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1168 "atomic_clear_" #name "():args"); \
1169 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1170 "atomic_clear_" #name "():ptr"); \
1171 atomic_clear_##name(ptr, val); \
1172 }
1173
1174 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1175 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1176 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
1177 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1178
1179 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \
1180 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1181 { \
1182 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1183 "atomic_fetchadd_" #name "():args"); \
1184 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1185 "atomic_fetchadd_" #name "():ptr"); \
1186 kmsan_init_ret(sizeof(type)); \
1187 return (atomic_fetchadd_##name(ptr, val)); \
1188 }
1189
1190 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
1191 type kmsan_atomic_readandclear_##name(volatile type *ptr) \
1192 { \
1193 kmsan_check_arg(sizeof(ptr), \
1194 "atomic_readandclear_" #name "():args"); \
1195 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1196 "atomic_readandclear_" #name "():ptr"); \
1197 kmsan_init_ret(sizeof(type)); \
1198 return (atomic_readandclear_##name(ptr)); \
1199 }
1200
1201 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
1202 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1203 { \
1204 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1205 "atomic_testandclear_" #name "():args"); \
1206 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1207 "atomic_testandclear_" #name "():ptr"); \
1208 kmsan_init_ret(sizeof(int)); \
1209 return (atomic_testandclear_##name(ptr, v)); \
1210 }
1211
1212 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
1213 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1214 { \
1215 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1216 "atomic_testandset_" #name "():args"); \
1217 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1218 "atomic_testandset_" #name "():ptr"); \
1219 kmsan_init_ret(sizeof(int)); \
1220 return (atomic_testandset_##name(ptr, v)); \
1221 }
1222
1223 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \
1224 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \
1225 { \
1226 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1227 "atomic_swap_" #name "():args"); \
1228 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1229 "atomic_swap_" #name "():ptr"); \
1230 kmsan_init_ret(sizeof(type)); \
1231 return (atomic_swap_##name(ptr, val)); \
1232 }
1233
1234 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1235 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \
1236 type nval) \
1237 { \
1238 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1239 sizeof(nval), "atomic_cmpset_" #name "():args"); \
1240 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1241 "atomic_cmpset_" #name "():ptr"); \
1242 kmsan_init_ret(sizeof(int)); \
1243 return (atomic_cmpset_##name(ptr, oval, nval)); \
1244 }
1245
1246 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1247 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1248 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
1249 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1250
1251 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1252 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1253 type nval) \
1254 { \
1255 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1256 sizeof(nval), "atomic_fcmpset_" #name "():args"); \
1257 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1258 "atomic_fcmpset_" #name "():ptr"); \
1259 kmsan_init_ret(sizeof(int)); \
1260 return (atomic_fcmpset_##name(ptr, oval, nval)); \
1261 }
1262
1263 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1264 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1265 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
1266 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1267
1268 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
1269 void kmsan_atomic_thread_fence_##name(void) \
1270 { \
1271 atomic_thread_fence_##name(); \
1272 }
1273
1274 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1275 type kmsan_atomic_load_##name(volatile type *ptr) \
1276 { \
1277 kmsan_check_arg(sizeof(ptr), \
1278 "atomic_load_" #name "():args"); \
1279 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1280 "atomic_load_" #name "():ptr"); \
1281 kmsan_init_ret(sizeof(type)); \
1282 return (atomic_load_##name(ptr)); \
1283 }
1284
1285 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \
1286 _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1287 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1288
1289 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \
1290 void kmsan_atomic_store_##name(volatile type *ptr, type val) \
1291 { \
1292 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1293 "atomic_store_" #name "():args"); \
1294 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \
1295 sizeof(type)); \
1296 atomic_store_##name(ptr, val); \
1297 }
1298
1299 #define MSAN_ATOMIC_FUNC_STORE(name, type) \
1300 _MSAN_ATOMIC_FUNC_STORE(name, type) \
1301 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1302
1303 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1304 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1305 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1306 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1307 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1308 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1309 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1310
1311 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1312 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1313 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1314 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1315 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1316 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1317 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1318
1319 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1320 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1321 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1322 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1323 MSAN_ATOMIC_FUNC_SET(int, u_int);
1324 MSAN_ATOMIC_FUNC_SET(long, u_long);
1325 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1326
1327 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1328 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1329 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1330 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1331 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1332 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1333 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1334
1335 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1336 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1337 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1338 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1339
1340 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1341 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1342 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1343 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1344 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1345
1346 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1347 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1348 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1349 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1350
1351 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1352 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1353 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1354 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1355
1356 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1357 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1358 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1359 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1360 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1361
1362 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1363 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1364 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1365 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1366 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1367 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1368 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1369
1370 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1371 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1372 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1373 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1374 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1375 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1376 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1377
1378 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1379 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1380 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1381 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1382 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1383 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1384 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1385 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1386 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1387 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1388
1389 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1390 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1391 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1392 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1393 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1394 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1395 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1396 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1397 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1398 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1399
1400 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1401 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1402 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1403 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1404
1405 void
1406 kmsan_atomic_interrupt_fence(void)
1407 {
1408 atomic_interrupt_fence();
1409 }
1410
1411 /* -------------------------------------------------------------------------- */
1412
1413 #include <sys/bus.h>
1414 #include <machine/bus.h>
1415 #include <sys/bus_san.h>
1416
1417 int
1418 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1419 int flags, bus_space_handle_t *handlep)
1420 {
1421 return (bus_space_map(tag, hnd, size, flags, handlep));
1422 }
1423
1424 void
1425 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1426 bus_size_t size)
1427 {
1428 bus_space_unmap(tag, hnd, size);
1429 }
1430
1431 int
1432 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1433 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1434 {
1435 return (bus_space_subregion(tag, hnd, offset, size, handlep));
1436 }
1437
1438 void
1439 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1440 bus_size_t size)
1441 {
1442 bus_space_free(tag, hnd, size);
1443 }
1444
1445 void
1446 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1447 bus_size_t offset, bus_size_t size, int flags)
1448 {
1449 bus_space_barrier(tag, hnd, offset, size, flags);
1450 }
1451
1452 /* XXXMJ x86-specific */
1453 #define MSAN_BUS_READ_FUNC(func, width, type) \
1454 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
1455 bus_space_handle_t hnd, bus_size_t offset) \
1456 { \
1457 type ret; \
1458 if ((tag) != X86_BUS_SPACE_IO) \
1459 kmsan_shadow_fill((uintptr_t)(hnd + offset), \
1460 KMSAN_STATE_INITED, (width)); \
1461 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1462 kmsan_init_ret(sizeof(type)); \
1463 return (ret); \
1464 } \
1465
1466 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \
1467 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1468 bus_space_handle_t hnd, bus_size_t size, type *buf, \
1469 bus_size_t count) \
1470 { \
1471 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \
1472 (width) * count); \
1473 bus_space_read_##func##_##width(tag, hnd, size, buf, \
1474 count); \
1475 }
1476
1477 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1478 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1479 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1480 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1481 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1482 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1483
1484 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1485 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1486 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1487 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1488 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1489 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1490
1491 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1492 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1493 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1494 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1495 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1496 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1497
1498 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1499
1500 #define MSAN_BUS_WRITE_FUNC(func, width, type) \
1501 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1502 bus_space_handle_t hnd, bus_size_t offset, type value) \
1503 { \
1504 bus_space_write##func##_##width(tag, hnd, offset, value);\
1505 } \
1506
1507 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
1508 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1509 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
1510 bus_size_t count) \
1511 { \
1512 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1513 "bus_space_write()"); \
1514 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1515 count); \
1516 }
1517
1518 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1519 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1520 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1521 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1522 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1523 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1524
1525 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1526 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1527 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1528 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1529 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1530 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1531
1532 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1533 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1534 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1535 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1536 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1537 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1538
1539 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1540
1541 #define MSAN_BUS_SET_FUNC(func, width, type) \
1542 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1543 bus_space_handle_t hnd, bus_size_t offset, type value, \
1544 bus_size_t count) \
1545 { \
1546 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1547 count); \
1548 }
1549
1550 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1551 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1552 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1553 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1554
1555 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1556 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1557 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1558 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1559
1560 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1561 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1562 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1563 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1564
1565 /* -------------------------------------------------------------------------- */
1566
1567 void
1568 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1569 {
1570 /*
1571 * Some drivers, e.g., nvme, use the same code path for loading device
1572 * read and write requests, and will thus specify both flags. In this
1573 * case we should not do any checking since it will generally lead to
1574 * false positives.
1575 */
1576 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1577 BUS_DMASYNC_PREWRITE) {
1578 switch (desc->md_type) {
1579 case MEMDESC_VADDR:
1580 kmsan_check(desc->u.md_vaddr, desc->md_opaque,
1581 "dmasync");
1582 break;
1583 case MEMDESC_BIO:
1584 kmsan_check_bio(desc->u.md_bio, "dmasync");
1585 break;
1586 case MEMDESC_MBUF:
1587 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1588 break;
1589 case MEMDESC_CCB:
1590 kmsan_check_ccb(desc->u.md_ccb, "dmasync");
1591 break;
1592 case 0:
1593 break;
1594 default:
1595 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1596 desc->md_type);
1597 }
1598 }
1599 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1600 switch (desc->md_type) {
1601 case MEMDESC_VADDR:
1602 kmsan_mark(desc->u.md_vaddr, desc->md_opaque,
1603 KMSAN_STATE_INITED);
1604 break;
1605 case MEMDESC_BIO:
1606 kmsan_mark_bio(desc->u.md_bio, KMSAN_STATE_INITED);
1607 break;
1608 case MEMDESC_MBUF:
1609 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1610 break;
1611 case MEMDESC_CCB:
1612 kmsan_mark_ccb(desc->u.md_ccb, KMSAN_STATE_INITED);
1613 break;
1614 case 0:
1615 break;
1616 default:
1617 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1618 desc->md_type);
1619 }
1620 }
1621 }
Cache object: a6699b581352048b39f0396d2addb1d7
|