FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_msan.c
1 /* $NetBSD: subr_msan.c,v 1.18 2022/10/26 23:38:09 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KMSAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.18 2022/10/26 23:38:09 riastradh Exp $");
33
34 #include <sys/param.h>
35 #include <sys/device.h>
36 #include <sys/kernel.h>
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/systm.h>
40 #include <sys/types.h>
41 #include <sys/kprintf.h>
42 #include <sys/kmem.h>
43 #include <sys/mbuf.h>
44 #include <sys/buf.h>
45 #include <sys/cpu.h>
46 #include <sys/msan.h>
47
48 #include <ddb/db_active.h>
49
50 static void kmsan_printf(const char *, ...);
51
52 void kmsan_init_arg(size_t);
53 void kmsan_init_ret(size_t);
54
55 #ifdef KMSAN_PANIC
56 #define REPORT panic
57 #else
58 #define REPORT kmsan_printf
59 #endif
60
61 /* -------------------------------------------------------------------------- */
62
63 /*
64 * Part of the compiler ABI.
65 */
66
67 typedef uint32_t msan_orig_t;
68
69 typedef struct {
70 uint8_t *shad;
71 msan_orig_t *orig;
72 } msan_meta_t;
73
74 #define MSAN_PARAM_SIZE 800
75 #define MSAN_RETVAL_SIZE 800
76 typedef struct {
77 uint8_t param[MSAN_PARAM_SIZE];
78 uint8_t retval[MSAN_RETVAL_SIZE];
79 uint8_t _va_arg[MSAN_PARAM_SIZE];
80 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
81 uint64_t va_arg_overflow_size;
82 msan_orig_t param_origin[MSAN_PARAM_SIZE];
83 msan_orig_t retval_origin;
84 msan_orig_t origin;
85 } msan_tls_t;
86
87 /* -------------------------------------------------------------------------- */
88
89 /* The MD code. */
90 #include <machine/msan.h>
91
92 /* -------------------------------------------------------------------------- */
93
94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
95 #define MSAN_NCONTEXT 16
96
97 typedef struct {
98 size_t ctx;
99 msan_tls_t tls[MSAN_NCONTEXT];
100 } msan_lwp_t;
101
102 static msan_tls_t dummy_tls;
103
104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
106 static msan_lwp_t msan_lwp0;
107 static bool kmsan_enabled __read_mostly;
108
109 /* -------------------------------------------------------------------------- */
110
111 static bool kmsan_reporting = false;
112
113 static inline void
114 kmsan_printf(const char *fmt, ...)
115 {
116 va_list ap;
117
118 va_start(ap, fmt);
119 kprintf(fmt, TOCONS, NULL, NULL, ap);
120 va_end(ap);
121 }
122
123 static inline const char *
124 kmsan_orig_name(int type)
125 {
126 switch (type) {
127 case KMSAN_TYPE_STACK:
128 return "Stack";
129 case KMSAN_TYPE_KMEM:
130 return "Kmem";
131 case KMSAN_TYPE_MALLOC:
132 return "Malloc";
133 case KMSAN_TYPE_POOL:
134 return "Pool";
135 case KMSAN_TYPE_UVM:
136 return "Uvm";
137 default:
138 return "Unknown";
139 }
140 }
141
142 /*
143 * The format of the string is: "----var@function". Parse it to display a nice
144 * warning.
145 */
146 static void
147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
148 {
149 const char *mod, *sym;
150 msan_orig_t *orig;
151 const char *typename;
152 char *var, *fn;
153 uintptr_t ptr;
154 char buf[128];
155 int type;
156 int s;
157
158 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
159 return;
160
161 kmsan_reporting = true;
162 __insn_barrier();
163
164 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
165 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
166
167 if (*orig == 0) {
168 REPORT("MSan: Uninitialized Memory In %s At Offset "
169 "%zu\n", hook, off);
170 goto out;
171 }
172
173 kmsan_md_orig_decode(*orig, &type, &ptr);
174 typename = kmsan_orig_name(type);
175
176 if (kmsan_md_is_pc(ptr)) {
177 s = pserialize_read_enter();
178 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) {
179 REPORT("MSan: Uninitialized %s Memory In %s "
180 "At Offset %zu/%zu, IP %p\n", typename, hook, off,
181 size, (void *)ptr);
182 } else {
183 REPORT("MSan: Uninitialized %s Memory In %s "
184 "At Offset %zu/%zu, From %s()\n", typename, hook,
185 off, size, sym);
186 }
187 pserialize_read_exit(s);
188 } else {
189 var = (char *)ptr + 4;
190 strlcpy(buf, var, sizeof(buf));
191 var = buf;
192 fn = __builtin_strchr(buf, '@');
193 *fn++ = '\0';
194 REPORT("MSan: Uninitialized %s Memory In %s At Offset "
195 "%zu, Variable '%s' From %s()\n", typename, hook, off,
196 var, fn);
197 }
198
199 out:
200 kmsan_md_unwind();
201 __insn_barrier();
202 kmsan_reporting = false;
203 }
204
205 static void
206 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
207 {
208 const char *mod, *sym;
209 const char *typename;
210 char *var, *fn;
211 uintptr_t ptr;
212 char buf[128];
213 int type;
214 int s;
215
216 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
217 return;
218
219 kmsan_reporting = true;
220 __insn_barrier();
221
222 if (orig == 0) {
223 REPORT("MSan: Uninitialized Variable In %p\n",
224 (void *)pc);
225 goto out;
226 }
227
228 kmsan_md_orig_decode(orig, &type, &ptr);
229 typename = kmsan_orig_name(type);
230
231 if (kmsan_md_is_pc(ptr)) {
232 s = pserialize_read_enter();
233 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) {
234 REPORT("MSan: Uninitialized %s Memory, "
235 "Origin %x\n", typename, orig);
236 } else {
237 REPORT("MSan: Uninitialized %s Memory "
238 "From %s()\n", typename, sym);
239 }
240 pserialize_read_exit(s);
241 } else {
242 var = (char *)ptr + 4;
243 strlcpy(buf, var, sizeof(buf));
244 var = buf;
245 fn = __builtin_strchr(buf, '@');
246 *fn++ = '\0';
247 REPORT("MSan: Uninitialized Variable '%s' From %s()\n",
248 var, fn);
249 }
250
251 out:
252 kmsan_md_unwind();
253 __insn_barrier();
254 kmsan_reporting = false;
255 }
256
257 /* -------------------------------------------------------------------------- */
258
259 static inline msan_meta_t
260 kmsan_meta_get(void *addr, size_t size)
261 {
262 msan_meta_t ret;
263
264 if (__predict_false(!kmsan_enabled)) {
265 ret.shad = msan_dummy_shad;
266 ret.orig = (msan_orig_t *)msan_dummy_orig;
267 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) {
268 ret.shad = msan_dummy_shad;
269 ret.orig = (msan_orig_t *)msan_dummy_orig;
270 } else {
271 ret.shad = (void *)kmsan_md_addr_to_shad(addr);
272 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
273 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3);
274 }
275
276 return ret;
277 }
278
279 static inline void
280 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size)
281 {
282 msan_orig_t *orig;
283 size_t i;
284
285 if (__predict_false(!kmsan_enabled))
286 return;
287 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
288 return;
289
290 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
291 size += ((uintptr_t)orig & 0x3);
292 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
293
294 for (i = 0; i < size; i += 4) {
295 orig[i / 4] = o;
296 }
297 }
298
299 static inline void
300 kmsan_shadow_fill(void *addr, uint8_t c, size_t size)
301 {
302 uint8_t *shad;
303
304 if (__predict_false(!kmsan_enabled))
305 return;
306 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
307 return;
308
309 shad = kmsan_md_addr_to_shad(addr);
310 __builtin_memset(shad, c, size);
311 }
312
313 static inline void
314 kmsan_meta_copy(void *dst, const void *src, size_t size)
315 {
316 uint8_t *orig_src, *orig_dst;
317 uint8_t *shad_src, *shad_dst;
318 msan_orig_t *_src, *_dst;
319 size_t i;
320
321 if (__predict_false(!kmsan_enabled))
322 return;
323 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst)))
324 return;
325 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) {
326 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size);
327 return;
328 }
329
330 shad_src = kmsan_md_addr_to_shad(src);
331 shad_dst = kmsan_md_addr_to_shad(dst);
332 __builtin_memmove(shad_dst, shad_src, size);
333
334 orig_src = kmsan_md_addr_to_orig(src);
335 orig_dst = kmsan_md_addr_to_orig(dst);
336 for (i = 0; i < size; i++) {
337 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3);
338 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3);
339 *_dst = *_src;
340 orig_src++;
341 orig_dst++;
342 }
343 }
344
345 static inline void
346 kmsan_shadow_check(const void *addr, size_t size, const char *hook)
347 {
348 uint8_t *shad;
349 size_t i;
350
351 if (__predict_false(!kmsan_enabled))
352 return;
353 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
354 return;
355
356 shad = kmsan_md_addr_to_shad(addr);
357 for (i = 0; i < size; i++) {
358 if (__predict_true(shad[i] == 0))
359 continue;
360 kmsan_report_hook((const char *)addr + i, size, i, hook);
361 break;
362 }
363 }
364
365 void
366 kmsan_init_arg(size_t n)
367 {
368 msan_lwp_t *lwp;
369 uint8_t *arg;
370
371 if (__predict_false(!kmsan_enabled))
372 return;
373 lwp = curlwp->l_kmsan;
374 arg = lwp->tls[lwp->ctx].param;
375 __builtin_memset(arg, 0, n);
376 }
377
378 void
379 kmsan_init_ret(size_t n)
380 {
381 msan_lwp_t *lwp;
382 uint8_t *arg;
383
384 if (__predict_false(!kmsan_enabled))
385 return;
386 lwp = curlwp->l_kmsan;
387 arg = lwp->tls[lwp->ctx].retval;
388 __builtin_memset(arg, 0, n);
389 }
390
391 static void
392 kmsan_check_arg(size_t size, const char *hook)
393 {
394 msan_lwp_t *lwp;
395 uint8_t *arg;
396 size_t i;
397
398 if (__predict_false(!kmsan_enabled))
399 return;
400 lwp = curlwp->l_kmsan;
401 arg = lwp->tls[lwp->ctx].param;
402
403 for (i = 0; i < size; i++) {
404 if (__predict_true(arg[i] == 0))
405 continue;
406 kmsan_report_hook((const char *)arg + i, size, i, hook);
407 break;
408 }
409 }
410
411 void
412 kmsan_lwp_alloc(struct lwp *l)
413 {
414 msan_lwp_t *lwp;
415
416 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t));
417 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP);
418 lwp->ctx = 1;
419
420 l->l_kmsan = lwp;
421 }
422
423 void
424 kmsan_lwp_free(struct lwp *l)
425 {
426 kmsan_init_arg(sizeof(void *) + sizeof(size_t));
427 kmem_free(l->l_kmsan, sizeof(msan_lwp_t));
428 }
429
430 void kmsan_intr_enter(void);
431 void kmsan_intr_leave(void);
432 void kmsan_softint(struct lwp *);
433
434 void
435 kmsan_intr_enter(void)
436 {
437 msan_lwp_t *lwp;
438
439 if (__predict_false(!kmsan_enabled))
440 return;
441 lwp = curlwp->l_kmsan;
442
443 lwp->ctx++;
444 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) {
445 kmsan_enabled = false;
446 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
447 }
448
449 kmsan_init_arg(sizeof(void *));
450 }
451
452 void
453 kmsan_intr_leave(void)
454 {
455 msan_lwp_t *lwp;
456
457 if (__predict_false(!kmsan_enabled))
458 return;
459 lwp = curlwp->l_kmsan;
460
461 if (__predict_false(lwp->ctx == 0)) {
462 kmsan_enabled = false;
463 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
464 }
465 lwp->ctx--;
466 }
467
468 void
469 kmsan_softint(struct lwp *l)
470 {
471 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int));
472 }
473
474 /* -------------------------------------------------------------------------- */
475
476 void
477 kmsan_shadow_map(void *addr, size_t size)
478 {
479 size_t npages, i;
480 vaddr_t va;
481
482 KASSERT((vaddr_t)addr % PAGE_SIZE == 0);
483 KASSERT(size % PAGE_SIZE == 0);
484
485 npages = size / PAGE_SIZE;
486
487 va = (vaddr_t)kmsan_md_addr_to_shad(addr);
488 for (i = 0; i < npages; i++) {
489 kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
490 }
491
492 va = (vaddr_t)kmsan_md_addr_to_orig(addr);
493 for (i = 0; i < npages; i++) {
494 kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
495 }
496 }
497
498 void
499 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc)
500 {
501 msan_orig_t orig;
502
503 orig = kmsan_md_orig_encode(type, pc);
504 kmsan_origin_fill(addr, orig, size);
505 }
506
507 void
508 kmsan_mark(void *addr, size_t size, uint8_t c)
509 {
510 kmsan_shadow_fill(addr, c, size);
511 }
512
513 void
514 kmsan_check_mbuf(void *buf)
515 {
516 struct mbuf *m = buf;
517
518 do {
519 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain");
520 } while ((m = m->m_next) != NULL);
521 }
522
523 void
524 kmsan_check_buf(void *buf)
525 {
526 buf_t *bp = buf;
527
528 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()");
529 }
530
531 void
532 kmsan_init(void *stack)
533 {
534 /* MD initialization. */
535 kmsan_md_init();
536
537 /* Map the stack. */
538 kmsan_shadow_map(stack, USPACE);
539
540 /* Initialize the TLS for curlwp. */
541 msan_lwp0.ctx = 1;
542 curlwp->l_kmsan = &msan_lwp0;
543
544 /* Now officially enabled. */
545 kmsan_enabled = true;
546 }
547
548 /* -------------------------------------------------------------------------- */
549
550 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
551 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
552
553 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size)
554 {
555 return kmsan_meta_get(addr, size);
556 }
557
558 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size)
559 {
560 return kmsan_meta_get(addr, size);
561 }
562
563 #define MSAN_META_FUNC(size) \
564 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
565 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
566 { \
567 return kmsan_meta_get(addr, size); \
568 } \
569 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
570 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
571 { \
572 return kmsan_meta_get(addr, size); \
573 }
574
575 MSAN_META_FUNC(1)
576 MSAN_META_FUNC(2)
577 MSAN_META_FUNC(4)
578 MSAN_META_FUNC(8)
579
580 void __msan_instrument_asm_store(void *, size_t);
581 msan_orig_t __msan_chain_origin(msan_orig_t);
582 void __msan_poison_alloca(void *, uint64_t, char *);
583 void __msan_unpoison_alloca(void *, uint64_t);
584 void __msan_warning(msan_orig_t);
585 msan_tls_t *__msan_get_context_state(void);
586
587 void __msan_instrument_asm_store(void *addr, size_t size)
588 {
589 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
590 }
591
592 msan_orig_t __msan_chain_origin(msan_orig_t origin)
593 {
594 return origin;
595 }
596
597 void __msan_poison_alloca(void *addr, uint64_t size, char *descr)
598 {
599 msan_orig_t orig;
600
601 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
602 kmsan_origin_fill(addr, orig, size);
603 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size);
604 }
605
606 void __msan_unpoison_alloca(void *addr, uint64_t size)
607 {
608 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
609 }
610
611 void __msan_warning(msan_orig_t origin)
612 {
613 if (__predict_false(!kmsan_enabled))
614 return;
615 kmsan_report_inline(origin, __RET_ADDR);
616 }
617
618 msan_tls_t *__msan_get_context_state(void)
619 {
620 msan_lwp_t *lwp;
621
622 if (__predict_false(!kmsan_enabled))
623 return &dummy_tls;
624 lwp = curlwp->l_kmsan;
625
626 return &lwp->tls[lwp->ctx];
627 }
628
629 /* -------------------------------------------------------------------------- */
630
631 /*
632 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
633 * initialized areas properly.
634 */
635
636 void *kmsan_memcpy(void *dst, const void *src, size_t len)
637 {
638 /* No kmsan_check_arg, because inlined. */
639 kmsan_init_ret(sizeof(void *));
640 if (__predict_true(len != 0)) {
641 kmsan_meta_copy(dst, src, len);
642 }
643 return __builtin_memcpy(dst, src, len);
644 }
645
646 int
647 kmsan_memcmp(const void *b1, const void *b2, size_t len)
648 {
649 const uint8_t *_b1 = b1, *_b2 = b2;
650 size_t i;
651
652 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
653 "memcmp():args");
654 kmsan_init_ret(sizeof(int));
655
656 for (i = 0; i < len; i++) {
657 if (*_b1 != *_b2) {
658 kmsan_shadow_check(b1, i + 1, "memcmp():arg1");
659 kmsan_shadow_check(b2, i + 1, "memcmp():arg2");
660 return *_b1 - *_b2;
661 }
662 _b1++, _b2++;
663 }
664
665 return 0;
666 }
667
668 void *kmsan_memset(void *dst, int c, size_t len)
669 {
670 /* No kmsan_check_arg, because inlined. */
671 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len);
672 kmsan_init_ret(sizeof(void *));
673 return __builtin_memset(dst, c, len);
674 }
675
676 void *kmsan_memmove(void *dst, const void *src, size_t len)
677 {
678 /* No kmsan_check_arg, because inlined. */
679 if (__predict_true(len != 0)) {
680 kmsan_meta_copy(dst, src, len);
681 }
682 kmsan_init_ret(sizeof(void *));
683 return __builtin_memmove(dst, src, len);
684 }
685
686 __strong_alias(__msan_memcpy, kmsan_memcpy)
687 __strong_alias(__msan_memset, kmsan_memset)
688 __strong_alias(__msan_memmove, kmsan_memmove)
689
690 char *
691 kmsan_strcpy(char *dst, const char *src)
692 {
693 const char *_src = src;
694 char *_dst = dst;
695 size_t len = 0;
696
697 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
698
699 while (1) {
700 len++;
701 *dst = *src;
702 if (*src == '\0')
703 break;
704 src++, dst++;
705 }
706
707 kmsan_shadow_check(_src, len, "strcpy():arg2");
708 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len);
709 kmsan_init_ret(sizeof(char *));
710 return _dst;
711 }
712
713 int
714 kmsan_strcmp(const char *s1, const char *s2)
715 {
716 const char *_s1 = s1, *_s2 = s2;
717 size_t len = 0;
718
719 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
720 kmsan_init_ret(sizeof(int));
721
722 while (1) {
723 len++;
724 if (*s1 != *s2)
725 break;
726 if (*s1 == '\0') {
727 kmsan_shadow_check(_s1, len, "strcmp():arg1");
728 kmsan_shadow_check(_s2, len, "strcmp():arg2");
729 return 0;
730 }
731 s1++, s2++;
732 }
733
734 kmsan_shadow_check(_s1, len, "strcmp():arg1");
735 kmsan_shadow_check(_s2, len, "strcmp():arg2");
736
737 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
738 }
739
740 size_t
741 kmsan_strlen(const char *str)
742 {
743 const char *s;
744
745 kmsan_check_arg(sizeof(str), "strlen():args");
746
747 s = str;
748 while (1) {
749 if (*s == '\0')
750 break;
751 s++;
752 }
753
754 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1");
755 kmsan_init_ret(sizeof(size_t));
756 return (s - str);
757 }
758
759 char *
760 kmsan_strcat(char *dst, const char *src)
761 {
762 size_t ldst, lsrc;
763 char *ret;
764
765 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args");
766
767 ldst = __builtin_strlen(dst);
768 lsrc = __builtin_strlen(src);
769 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1");
770 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2");
771 ret = __builtin_strcat(dst, src);
772 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1);
773
774 kmsan_init_ret(sizeof(char *));
775 return ret;
776 }
777
778 char *
779 kmsan_strchr(const char *s, int c)
780 {
781 char *ret;
782
783 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args");
784 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1");
785 ret = __builtin_strchr(s, c);
786
787 kmsan_init_ret(sizeof(char *));
788 return ret;
789 }
790
791 char *
792 kmsan_strrchr(const char *s, int c)
793 {
794 char *ret;
795
796 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args");
797 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1");
798 ret = __builtin_strrchr(s, c);
799
800 kmsan_init_ret(sizeof(char *));
801 return ret;
802 }
803
804 #undef kcopy
805 #undef copyin
806 #undef copyout
807 #undef copyinstr
808 #undef copyoutstr
809
810 int kmsan_kcopy(const void *, void *, size_t);
811 int kmsan_copyin(const void *, void *, size_t);
812 int kmsan_copyout(const void *, void *, size_t);
813 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
814 int kmsan_copyoutstr(const void *, void *, size_t, size_t *);
815
816 int kcopy(const void *, void *, size_t);
817 int copyin(const void *, void *, size_t);
818 int copyout(const void *, void *, size_t);
819 int copyinstr(const void *, void *, size_t, size_t *);
820 int copyoutstr(const void *, void *, size_t, size_t *);
821
822 int
823 kmsan_kcopy(const void *src, void *dst, size_t len)
824 {
825 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len),
826 "kcopy():args");
827 if (__predict_true(len != 0)) {
828 kmsan_meta_copy(dst, src, len);
829 }
830 kmsan_init_ret(sizeof(int));
831 return kcopy(src, dst, len);
832 }
833
834 int
835 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
836 {
837 int ret;
838
839 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
840 "copyin():args");
841 ret = copyin(uaddr, kaddr, len);
842 if (ret == 0)
843 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len);
844 kmsan_init_ret(sizeof(int));
845
846 return ret;
847 }
848
849 int
850 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
851 {
852 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
853 "copyout():args");
854 kmsan_shadow_check(kaddr, len, "copyout():arg1");
855 kmsan_init_ret(sizeof(int));
856 return copyout(kaddr, uaddr, len);
857 }
858
859 int
860 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
861 {
862 size_t _done;
863 int ret;
864
865 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
866 sizeof(len) + sizeof(done), "copyinstr():args");
867 ret = copyinstr(uaddr, kaddr, len, &_done);
868 if (ret == 0 || ret == ENAMETOOLONG)
869 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done);
870 if (done != NULL) {
871 *done = _done;
872 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
873 }
874 kmsan_init_ret(sizeof(int));
875
876 return ret;
877 }
878
879 int
880 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
881 {
882 size_t _done;
883 int ret;
884
885 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) +
886 sizeof(len) + sizeof(done), "copyoutstr():args");
887 ret = copyoutstr(kaddr, uaddr, len, &_done);
888 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1");
889 if (done != NULL) {
890 *done = _done;
891 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
892 }
893 kmsan_init_ret(sizeof(int));
894
895 return ret;
896 }
897
898 /* -------------------------------------------------------------------------- */
899
900 #undef _ucas_32
901 #undef _ucas_32_mp
902 #undef _ucas_64
903 #undef _ucas_64_mp
904 #undef _ufetch_8
905 #undef _ufetch_16
906 #undef _ufetch_32
907 #undef _ufetch_64
908 #undef _ustore_8
909 #undef _ustore_16
910 #undef _ustore_32
911 #undef _ustore_64
912
913 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
914 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
915 int
916 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
917 uint32_t *ret)
918 {
919 int _ret;
920 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
921 sizeof(new) + sizeof(ret), "ucas_32():args");
922 _ret = _ucas_32(uaddr, old, new, ret);
923 if (_ret == 0)
924 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
925 kmsan_init_ret(sizeof(int));
926 return _ret;
927 }
928
929 #ifdef __HAVE_UCAS_MP
930 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
931 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
932 int
933 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
934 uint32_t *ret)
935 {
936 int _ret;
937 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
938 sizeof(new) + sizeof(ret), "ucas_32_mp():args");
939 _ret = _ucas_32_mp(uaddr, old, new, ret);
940 if (_ret == 0)
941 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
942 kmsan_init_ret(sizeof(int));
943 return _ret;
944 }
945 #endif
946
947 #ifdef _LP64
948 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
949 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
950 int
951 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
952 uint64_t *ret)
953 {
954 int _ret;
955 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
956 sizeof(new) + sizeof(ret), "ucas_64():args");
957 _ret = _ucas_64(uaddr, old, new, ret);
958 if (_ret == 0)
959 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
960 kmsan_init_ret(sizeof(int));
961 return _ret;
962 }
963
964 #ifdef __HAVE_UCAS_MP
965 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
966 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
967 int
968 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
969 uint64_t *ret)
970 {
971 int _ret;
972 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
973 sizeof(new) + sizeof(ret), "ucas_64_mp():args");
974 _ret = _ucas_64_mp(uaddr, old, new, ret);
975 if (_ret == 0)
976 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
977 kmsan_init_ret(sizeof(int));
978 return _ret;
979 }
980 #endif
981 #endif
982
983 int _ufetch_8(const uint8_t *, uint8_t *);
984 int kmsan__ufetch_8(const uint8_t *, uint8_t *);
985 int
986 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
987 {
988 int _ret;
989 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args");
990 _ret = _ufetch_8(uaddr, valp);
991 if (_ret == 0)
992 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
993 kmsan_init_ret(sizeof(int));
994 return _ret;
995 }
996
997 int _ufetch_16(const uint16_t *, uint16_t *);
998 int kmsan__ufetch_16(const uint16_t *, uint16_t *);
999 int
1000 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
1001 {
1002 int _ret;
1003 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args");
1004 _ret = _ufetch_16(uaddr, valp);
1005 if (_ret == 0)
1006 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1007 kmsan_init_ret(sizeof(int));
1008 return _ret;
1009 }
1010
1011 int _ufetch_32(const uint32_t *, uint32_t *);
1012 int kmsan__ufetch_32(const uint32_t *, uint32_t *);
1013 int
1014 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
1015 {
1016 int _ret;
1017 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args");
1018 _ret = _ufetch_32(uaddr, valp);
1019 if (_ret == 0)
1020 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1021 kmsan_init_ret(sizeof(int));
1022 return _ret;
1023 }
1024
1025 #ifdef _LP64
1026 int _ufetch_64(const uint64_t *, uint64_t *);
1027 int kmsan__ufetch_64(const uint64_t *, uint64_t *);
1028 int
1029 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
1030 {
1031 int _ret;
1032 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args");
1033 _ret = _ufetch_64(uaddr, valp);
1034 if (_ret == 0)
1035 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1036 kmsan_init_ret(sizeof(int));
1037 return _ret;
1038 }
1039 #endif
1040
1041 int _ustore_8(uint8_t *, uint8_t);
1042 int kmsan__ustore_8(uint8_t *, uint8_t);
1043 int
1044 kmsan__ustore_8(uint8_t *uaddr, uint8_t val)
1045 {
1046 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args");
1047 kmsan_init_ret(sizeof(int));
1048 return _ustore_8(uaddr, val);
1049 }
1050
1051 int _ustore_16(uint16_t *, uint16_t);
1052 int kmsan__ustore_16(uint16_t *, uint16_t);
1053 int
1054 kmsan__ustore_16(uint16_t *uaddr, uint16_t val)
1055 {
1056 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args");
1057 kmsan_init_ret(sizeof(int));
1058 return _ustore_16(uaddr, val);
1059 }
1060
1061 int _ustore_32(uint32_t *, uint32_t);
1062 int kmsan__ustore_32(uint32_t *, uint32_t);
1063 int
1064 kmsan__ustore_32(uint32_t *uaddr, uint32_t val)
1065 {
1066 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args");
1067 kmsan_init_ret(sizeof(int));
1068 return _ustore_32(uaddr, val);
1069 }
1070
1071 #ifdef _LP64
1072 int _ustore_64(uint64_t *, uint64_t);
1073 int kmsan__ustore_64(uint64_t *, uint64_t);
1074 int
1075 kmsan__ustore_64(uint64_t *uaddr, uint64_t val)
1076 {
1077 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args");
1078 kmsan_init_ret(sizeof(int));
1079 return _ustore_64(uaddr, val);
1080 }
1081 #endif
1082
1083 /* -------------------------------------------------------------------------- */
1084
1085 #undef atomic_add_32
1086 #undef atomic_add_int
1087 #undef atomic_add_long
1088 #undef atomic_add_ptr
1089 #undef atomic_add_64
1090 #undef atomic_add_32_nv
1091 #undef atomic_add_int_nv
1092 #undef atomic_add_long_nv
1093 #undef atomic_add_ptr_nv
1094 #undef atomic_add_64_nv
1095 #undef atomic_and_32
1096 #undef atomic_and_uint
1097 #undef atomic_and_ulong
1098 #undef atomic_and_64
1099 #undef atomic_and_32_nv
1100 #undef atomic_and_uint_nv
1101 #undef atomic_and_ulong_nv
1102 #undef atomic_and_64_nv
1103 #undef atomic_or_32
1104 #undef atomic_or_uint
1105 #undef atomic_or_ulong
1106 #undef atomic_or_64
1107 #undef atomic_or_32_nv
1108 #undef atomic_or_uint_nv
1109 #undef atomic_or_ulong_nv
1110 #undef atomic_or_64_nv
1111 #undef atomic_cas_32
1112 #undef atomic_cas_uint
1113 #undef atomic_cas_ulong
1114 #undef atomic_cas_ptr
1115 #undef atomic_cas_64
1116 #undef atomic_cas_32_ni
1117 #undef atomic_cas_uint_ni
1118 #undef atomic_cas_ulong_ni
1119 #undef atomic_cas_ptr_ni
1120 #undef atomic_cas_64_ni
1121 #undef atomic_swap_32
1122 #undef atomic_swap_uint
1123 #undef atomic_swap_ulong
1124 #undef atomic_swap_ptr
1125 #undef atomic_swap_64
1126 #undef atomic_dec_32
1127 #undef atomic_dec_uint
1128 #undef atomic_dec_ulong
1129 #undef atomic_dec_ptr
1130 #undef atomic_dec_64
1131 #undef atomic_dec_32_nv
1132 #undef atomic_dec_uint_nv
1133 #undef atomic_dec_ulong_nv
1134 #undef atomic_dec_ptr_nv
1135 #undef atomic_dec_64_nv
1136 #undef atomic_inc_32
1137 #undef atomic_inc_uint
1138 #undef atomic_inc_ulong
1139 #undef atomic_inc_ptr
1140 #undef atomic_inc_64
1141 #undef atomic_inc_32_nv
1142 #undef atomic_inc_uint_nv
1143 #undef atomic_inc_ulong_nv
1144 #undef atomic_inc_ptr_nv
1145 #undef atomic_inc_64_nv
1146
1147 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
1148 void atomic_add_##name(volatile targ1 *, targ2); \
1149 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \
1150 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
1151 { \
1152 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1153 "atomic_add_" #name "():args"); \
1154 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1155 "atomic_add_" #name "():arg1"); \
1156 atomic_add_##name(ptr, val); \
1157 } \
1158 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
1159 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
1160 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
1161 { \
1162 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1163 "atomic_add_" #name "_nv():args"); \
1164 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1165 "atomic_add_" #name "_nv():arg1"); \
1166 kmsan_init_ret(sizeof(tret)); \
1167 return atomic_add_##name##_nv(ptr, val); \
1168 }
1169
1170 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
1171 void atomic_and_##name(volatile targ1 *, targ2); \
1172 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \
1173 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
1174 { \
1175 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1176 "atomic_and_" #name "():args"); \
1177 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1178 "atomic_and_" #name "():arg1"); \
1179 atomic_and_##name(ptr, val); \
1180 } \
1181 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
1182 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
1183 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
1184 { \
1185 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1186 "atomic_and_" #name "_nv():args"); \
1187 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1188 "atomic_and_" #name "_nv():arg1"); \
1189 kmsan_init_ret(sizeof(tret)); \
1190 return atomic_and_##name##_nv(ptr, val); \
1191 }
1192
1193 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
1194 void atomic_or_##name(volatile targ1 *, targ2); \
1195 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \
1196 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
1197 { \
1198 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1199 "atomic_or_" #name "():args"); \
1200 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1201 "atomic_or_" #name "():arg1"); \
1202 atomic_or_##name(ptr, val); \
1203 } \
1204 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
1205 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
1206 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
1207 { \
1208 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1209 "atomic_or_" #name "_nv():args"); \
1210 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1211 "atomic_or_" #name "_nv():arg1"); \
1212 kmsan_init_ret(sizeof(tret)); \
1213 return atomic_or_##name##_nv(ptr, val); \
1214 }
1215
1216 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
1217 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1218 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1219 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
1220 { \
1221 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1222 "atomic_cas_" #name "():args"); \
1223 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1224 "atomic_cas_" #name "():arg1"); \
1225 kmsan_init_ret(sizeof(tret)); \
1226 return atomic_cas_##name(ptr, exp, new); \
1227 } \
1228 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1229 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1230 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
1231 { \
1232 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1233 "atomic_cas_" #name "_ni():args"); \
1234 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1235 "atomic_cas_" #name "_ni():arg1"); \
1236 kmsan_init_ret(sizeof(tret)); \
1237 return atomic_cas_##name##_ni(ptr, exp, new); \
1238 }
1239
1240 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
1241 tret atomic_swap_##name(volatile targ1 *, targ2); \
1242 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \
1243 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
1244 { \
1245 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1246 "atomic_swap_" #name "():args"); \
1247 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1248 "atomic_swap_" #name "():arg1"); \
1249 kmsan_init_ret(sizeof(tret)); \
1250 return atomic_swap_##name(ptr, val); \
1251 }
1252
1253 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
1254 void atomic_dec_##name(volatile targ1 *); \
1255 void kmsan_atomic_dec_##name(volatile targ1 *); \
1256 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \
1257 { \
1258 kmsan_check_arg(sizeof(ptr), \
1259 "atomic_dec_" #name "():args"); \
1260 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1261 "atomic_dec_" #name "():arg1"); \
1262 atomic_dec_##name(ptr); \
1263 } \
1264 tret atomic_dec_##name##_nv(volatile targ1 *); \
1265 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \
1266 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
1267 { \
1268 kmsan_check_arg(sizeof(ptr), \
1269 "atomic_dec_" #name "_nv():args"); \
1270 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1271 "atomic_dec_" #name "_nv():arg1"); \
1272 kmsan_init_ret(sizeof(tret)); \
1273 return atomic_dec_##name##_nv(ptr); \
1274 }
1275
1276 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
1277 void atomic_inc_##name(volatile targ1 *); \
1278 void kmsan_atomic_inc_##name(volatile targ1 *); \
1279 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \
1280 { \
1281 kmsan_check_arg(sizeof(ptr), \
1282 "atomic_inc_" #name "():args"); \
1283 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1284 "atomic_inc_" #name "():arg1"); \
1285 atomic_inc_##name(ptr); \
1286 } \
1287 tret atomic_inc_##name##_nv(volatile targ1 *); \
1288 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \
1289 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
1290 { \
1291 kmsan_check_arg(sizeof(ptr), \
1292 "atomic_inc_" #name "_nv():args"); \
1293 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1294 "atomic_inc_" #name "_nv():arg1"); \
1295 kmsan_init_ret(sizeof(tret)); \
1296 return atomic_inc_##name##_nv(ptr); \
1297 }
1298
1299 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
1300 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
1301 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
1302 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
1303 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
1304
1305 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
1306 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
1307 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
1308 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
1309
1310 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
1311 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
1312 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
1313 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
1314
1315 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
1316 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
1317 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
1318 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
1319 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
1320
1321 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
1322 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
1323 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
1324 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
1325 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
1326
1327 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
1328 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
1329 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
1330 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
1331 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
1332
1333 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
1334 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
1335 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
1336 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
1337 MSAN_ATOMIC_FUNC_INC(ptr, void *, void);
1338
1339 /* -------------------------------------------------------------------------- */
1340
1341 #include <sys/bus.h>
1342
1343 #undef bus_space_read_multi_1
1344 #undef bus_space_read_multi_2
1345 #undef bus_space_read_multi_4
1346 #undef bus_space_read_multi_8
1347 #undef bus_space_read_multi_stream_1
1348 #undef bus_space_read_multi_stream_2
1349 #undef bus_space_read_multi_stream_4
1350 #undef bus_space_read_multi_stream_8
1351 #undef bus_space_read_region_1
1352 #undef bus_space_read_region_2
1353 #undef bus_space_read_region_4
1354 #undef bus_space_read_region_8
1355 #undef bus_space_read_region_stream_1
1356 #undef bus_space_read_region_stream_2
1357 #undef bus_space_read_region_stream_4
1358 #undef bus_space_read_region_stream_8
1359
1360 #define MSAN_BUS_READ_FUNC(bytes, bits) \
1361 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1362 bus_size_t, uint##bits##_t *, bus_size_t); \
1363 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
1364 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1365 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
1366 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1367 bus_size_t count) \
1368 { \
1369 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1370 sizeof(uint##bits##_t) * count); \
1371 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
1372 } \
1373 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1374 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1375 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1376 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1377 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
1378 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1379 bus_size_t count) \
1380 { \
1381 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1382 sizeof(uint##bits##_t) * count); \
1383 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
1384 } \
1385 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
1386 bus_size_t, uint##bits##_t *, bus_size_t); \
1387 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \
1388 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1389 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
1390 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1391 bus_size_t count) \
1392 { \
1393 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1394 sizeof(uint##bits##_t) * count); \
1395 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
1396 } \
1397 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1398 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1399 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1400 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1401 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
1402 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1403 bus_size_t count) \
1404 { \
1405 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1406 sizeof(uint##bits##_t) * count); \
1407 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1408 }
1409
1410 MSAN_BUS_READ_FUNC(1, 8)
1411 MSAN_BUS_READ_FUNC(2, 16)
1412 MSAN_BUS_READ_FUNC(4, 32)
1413 MSAN_BUS_READ_FUNC(8, 64)
1414
1415 #undef bus_space_write_multi_1
1416 #undef bus_space_write_multi_2
1417 #undef bus_space_write_multi_4
1418 #undef bus_space_write_multi_8
1419 #undef bus_space_write_multi_stream_1
1420 #undef bus_space_write_multi_stream_2
1421 #undef bus_space_write_multi_stream_4
1422 #undef bus_space_write_multi_stream_8
1423 #undef bus_space_write_region_1
1424 #undef bus_space_write_region_2
1425 #undef bus_space_write_region_4
1426 #undef bus_space_write_region_8
1427 #undef bus_space_write_region_stream_1
1428 #undef bus_space_write_region_stream_2
1429 #undef bus_space_write_region_stream_4
1430 #undef bus_space_write_region_stream_8
1431
1432 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \
1433 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1434 bus_size_t, const uint##bits##_t *, bus_size_t); \
1435 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1436 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1437 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1438 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1439 bus_size_t count) \
1440 { \
1441 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1442 "bus_space_write()"); \
1443 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1444 } \
1445 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1446 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1447 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1448 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1449 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1450 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1451 bus_size_t count) \
1452 { \
1453 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1454 "bus_space_write()"); \
1455 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1456 } \
1457 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1458 bus_size_t, const uint##bits##_t *, bus_size_t); \
1459 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \
1460 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1461 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1462 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1463 bus_size_t count) \
1464 { \
1465 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1466 "bus_space_write()"); \
1467 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1468 } \
1469 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1470 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1471 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1472 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1473 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1474 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1475 bus_size_t count) \
1476 { \
1477 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1478 "bus_space_write()"); \
1479 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1480 }
1481
1482 MSAN_BUS_WRITE_FUNC(1, 8)
1483 MSAN_BUS_WRITE_FUNC(2, 16)
1484 MSAN_BUS_WRITE_FUNC(4, 32)
1485 MSAN_BUS_WRITE_FUNC(8, 64)
1486
1487 /* -------------------------------------------------------------------------- */
1488
1489 #include <sys/mbuf.h>
1490
1491 static void
1492 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1493 bool init, uintptr_t pc)
1494 {
1495 if (init) {
1496 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len);
1497 } else {
1498 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp");
1499 }
1500 }
1501
1502 static void
1503 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1504 bool init, uintptr_t pc)
1505 {
1506 bus_addr_t minlen;
1507
1508 for (; m != NULL && len != 0; m = m->m_next) {
1509 if (offset >= m->m_len) {
1510 offset -= m->m_len;
1511 continue;
1512 }
1513
1514 minlen = MIN(len, m->m_len - offset);
1515
1516 if (init) {
1517 kmsan_shadow_fill(mtod(m, char *) + offset,
1518 KMSAN_STATE_INITED, minlen);
1519 } else {
1520 kmsan_shadow_check(mtod(m, char *) + offset,
1521 minlen, "MbufDmaSyncOp");
1522 }
1523
1524 offset = 0;
1525 len -= minlen;
1526 }
1527 }
1528
1529 static void
1530 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1531 bool init, uintptr_t pc)
1532 {
1533 bus_size_t minlen, resid;
1534 struct iovec *iov;
1535 int i;
1536
1537 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
1538 return;
1539
1540 resid = uio->uio_resid;
1541 iov = uio->uio_iov;
1542
1543 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1544 minlen = MIN(resid, iov[i].iov_len);
1545
1546 if (init) {
1547 kmsan_shadow_fill(iov[i].iov_base,
1548 KMSAN_STATE_INITED, minlen);
1549 } else {
1550 kmsan_shadow_check(iov[i].iov_base, minlen,
1551 "UioDmaSyncOp");
1552 }
1553
1554 resid -= minlen;
1555 }
1556 }
1557
1558 void
1559 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1560 {
1561 bool init;
1562
1563 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0)
1564 return;
1565 init = (ops & BUS_DMASYNC_POSTREAD) != 0;
1566
1567 switch (map->dm_buftype) {
1568 case KMSAN_DMA_LINEAR:
1569 kmsan_dma_sync_linear(map->dm_buf, offset, len, init,
1570 __RET_ADDR);
1571 break;
1572 case KMSAN_DMA_MBUF:
1573 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init,
1574 __RET_ADDR);
1575 break;
1576 case KMSAN_DMA_UIO:
1577 kmsan_dma_sync_uio(map->dm_buf, offset, len, init,
1578 __RET_ADDR);
1579 break;
1580 case KMSAN_DMA_RAW:
1581 break;
1582 default:
1583 panic("%s: impossible", __func__);
1584 }
1585 }
1586
1587 void
1588 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1589 {
1590 map->dm_buf = buf;
1591 map->dm_buflen = buflen;
1592 map->dm_buftype = type;
1593 }
Cache object: c45fba1b99f7a80bef1d3361ed836f71
|