FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_csan.c
1 /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 * Copyright (c) 2019 Andrew Turner
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Maxime Villard.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #define SAN_RUNTIME
34
35 #include "opt_ddb.h"
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39
40 #include <sys/param.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/conf.h>
44 #include <sys/cpu.h>
45 #include <sys/csan.h>
46 #include <sys/proc.h>
47 #include <sys/smp.h>
48 #include <sys/systm.h>
49
50 #include <ddb/ddb.h>
51 #include <ddb/db_sym.h>
52
53 #ifdef KCSAN_PANIC
54 #define REPORT panic
55 #else
56 #define REPORT printf
57 #endif
58
59 typedef struct {
60 uintptr_t addr;
61 uint32_t size;
62 bool write:1;
63 bool atomic:1;
64 uintptr_t pc;
65 } csan_cell_t;
66
67 typedef struct {
68 bool inited;
69 uint32_t cnt;
70 csan_cell_t cell;
71 } csan_cpu_t;
72
73 static csan_cpu_t kcsan_cpus[MAXCPU];
74 static bool kcsan_enabled __read_mostly;
75
76 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
77
78 #define KCSAN_NACCESSES 1024
79 #define KCSAN_DELAY 10 /* 10 microseconds */
80
81 /* -------------------------------------------------------------------------- */
82
83 /* The MD code. */
84 #include <machine/csan.h>
85
86 /* -------------------------------------------------------------------------- */
87
88 static void
89 kcsan_enable(void *dummy __unused)
90 {
91
92 printf("Enabling KCSCAN, expect reduced performance.\n");
93 kcsan_enabled = true;
94 }
95 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
96
97 void
98 kcsan_cpu_init(u_int cpu)
99 {
100 kcsan_cpus[cpu].inited = true;
101 }
102
103 /* -------------------------------------------------------------------------- */
104
105 static inline void
106 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
107 {
108 const char *newsym, *oldsym;
109 #ifdef DDB
110 c_db_sym_t sym;
111 db_expr_t offset;
112
113 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
114 db_symbol_values(sym, &newsym, NULL);
115
116 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
117 db_symbol_values(sym, &oldsym, NULL);
118 #else
119 newsym = "";
120 oldsym = "";
121 #endif
122 REPORT("CSan: Racy Access "
123 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
124 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
125 newcpu,
126 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
127 (void *)new->addr, new->size, (void *)new->pc, newsym,
128 oldcpu,
129 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
130 (void *)old->addr, old->size, (void *)old->pc, oldsym);
131 kcsan_md_unwind();
132 }
133
134 static inline bool
135 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
136 {
137 if (new->write && !new->atomic)
138 return false;
139 if (old->write && !old->atomic)
140 return false;
141 return true;
142 }
143
144 static inline void
145 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
146 {
147 csan_cell_t old, new;
148 csan_cpu_t *cpu;
149 uint64_t intr;
150 size_t i;
151
152 if (__predict_false(!kcsan_enabled))
153 return;
154 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr)))
155 return;
156 if (KERNEL_PANICKED())
157 return;
158
159 new.addr = addr;
160 new.size = size;
161 new.write = write;
162 new.atomic = atomic;
163 new.pc = pc;
164
165 CPU_FOREACH(i) {
166 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
167
168 if (old.addr + old.size <= new.addr)
169 continue;
170 if (new.addr + new.size <= old.addr)
171 continue;
172 if (__predict_true(!old.write && !new.write))
173 continue;
174 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
175 continue;
176
177 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
178 break;
179 }
180
181 if (__predict_false(!kcsan_md_is_avail()))
182 return;
183
184 kcsan_md_disable_intrs(&intr);
185
186 cpu = &kcsan_cpus[PCPU_GET(cpuid)];
187 if (__predict_false(!cpu->inited))
188 goto out;
189 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
190 if (__predict_true(cpu->cnt != 0))
191 goto out;
192
193 __builtin_memcpy(&cpu->cell, &new, sizeof(new));
194 kcsan_md_delay(KCSAN_DELAY);
195 __builtin_memset(&cpu->cell, 0, sizeof(new));
196
197 out:
198 kcsan_md_enable_intrs(&intr);
199 }
200
201 #define CSAN_READ(size) \
202 void __tsan_read##size(uintptr_t); \
203 void __tsan_read##size(uintptr_t addr) \
204 { \
205 kcsan_access(addr, size, false, false, __RET_ADDR); \
206 } \
207 void __tsan_unaligned_read##size(uintptr_t); \
208 void __tsan_unaligned_read##size(uintptr_t addr) \
209 { \
210 kcsan_access(addr, size, false, false, __RET_ADDR); \
211 }
212
213 CSAN_READ(1)
214 CSAN_READ(2)
215 CSAN_READ(4)
216 CSAN_READ(8)
217 CSAN_READ(16)
218
219 #define CSAN_WRITE(size) \
220 void __tsan_write##size(uintptr_t); \
221 void __tsan_write##size(uintptr_t addr) \
222 { \
223 kcsan_access(addr, size, true, false, __RET_ADDR); \
224 } \
225 void __tsan_unaligned_write##size(uintptr_t); \
226 void __tsan_unaligned_write##size(uintptr_t addr) \
227 { \
228 kcsan_access(addr, size, true, false, __RET_ADDR); \
229 }
230
231 CSAN_WRITE(1)
232 CSAN_WRITE(2)
233 CSAN_WRITE(4)
234 CSAN_WRITE(8)
235 CSAN_WRITE(16)
236
237 void __tsan_read_range(uintptr_t, size_t);
238 void __tsan_write_range(uintptr_t, size_t);
239
240 void
241 __tsan_read_range(uintptr_t addr, size_t size)
242 {
243 kcsan_access(addr, size, false, false, __RET_ADDR);
244 }
245
246 void
247 __tsan_write_range(uintptr_t addr, size_t size)
248 {
249 kcsan_access(addr, size, true, false, __RET_ADDR);
250 }
251
252 void __tsan_init(void);
253 void __tsan_func_entry(void *);
254 void __tsan_func_exit(void);
255
256 void
257 __tsan_init(void)
258 {
259 }
260
261 void
262 __tsan_func_entry(void *call_pc)
263 {
264 }
265
266 void
267 __tsan_func_exit(void)
268 {
269 }
270
271 /* -------------------------------------------------------------------------- */
272
273 void *
274 kcsan_memcpy(void *dst, const void *src, size_t len)
275 {
276 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
277 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
278 return __builtin_memcpy(dst, src, len);
279 }
280
281 int
282 kcsan_memcmp(const void *b1, const void *b2, size_t len)
283 {
284 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
285 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
286 return __builtin_memcmp(b1, b2, len);
287 }
288
289 void *
290 kcsan_memset(void *b, int c, size_t len)
291 {
292 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
293 return __builtin_memset(b, c, len);
294 }
295
296 void *
297 kcsan_memmove(void *dst, const void *src, size_t len)
298 {
299 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
300 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
301 return __builtin_memmove(dst, src, len);
302 }
303
304 char *
305 kcsan_strcpy(char *dst, const char *src)
306 {
307 char *save = dst;
308
309 while (1) {
310 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
311 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
312 *dst = *src;
313 if (*src == '\0')
314 break;
315 src++, dst++;
316 }
317
318 return save;
319 }
320
321 int
322 kcsan_strcmp(const char *s1, const char *s2)
323 {
324 while (1) {
325 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
326 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
327 if (*s1 != *s2)
328 break;
329 if (*s1 == '\0')
330 return 0;
331 s1++, s2++;
332 }
333
334 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
335 }
336
337 size_t
338 kcsan_strlen(const char *str)
339 {
340 const char *s;
341
342 s = str;
343 while (1) {
344 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
345 if (*s == '\0')
346 break;
347 s++;
348 }
349
350 return (s - str);
351 }
352
353 int
354 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
355 {
356 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
357 return copyin(uaddr, kaddr, len);
358 }
359
360 int
361 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
362 {
363 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
364 return copyinstr(uaddr, kaddr, len, done);
365 }
366
367 int
368 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
369 {
370 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
371 return copyout(kaddr, uaddr, len);
372 }
373
374 /* -------------------------------------------------------------------------- */
375
376 #include <machine/atomic.h>
377 #include <sys/atomic_san.h>
378
379 #define _CSAN_ATOMIC_FUNC_ADD(name, type) \
380 void kcsan_atomic_add_##name(volatile type *ptr, type val) \
381 { \
382 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
383 __RET_ADDR); \
384 atomic_add_##name(ptr, val); \
385 }
386
387 #define CSAN_ATOMIC_FUNC_ADD(name, type) \
388 _CSAN_ATOMIC_FUNC_ADD(name, type) \
389 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
390 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
391
392 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
393 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
394 { \
395 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
396 __RET_ADDR); \
397 atomic_clear_##name(ptr, val); \
398 }
399
400 #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
401 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
402 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
403 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
404
405 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
406 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
407 type val2) \
408 { \
409 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
410 __RET_ADDR); \
411 return (atomic_cmpset_##name(ptr, val1, val2)); \
412 }
413
414 #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
415 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
416 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
417 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
418
419 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
420 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
421 type val2) \
422 { \
423 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
424 __RET_ADDR); \
425 return (atomic_fcmpset_##name(ptr, val1, val2)); \
426 }
427
428 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
429 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
430 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
431 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
432
433 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
434 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
435 { \
436 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
437 __RET_ADDR); \
438 return (atomic_fetchadd_##name(ptr, val)); \
439 }
440
441 #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
442 type kcsan_atomic_load_##name(volatile type *ptr) \
443 { \
444 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
445 __RET_ADDR); \
446 return (atomic_load_##name(ptr)); \
447 }
448
449 #define CSAN_ATOMIC_FUNC_LOAD(name, type) \
450 _CSAN_ATOMIC_FUNC_LOAD(name, type) \
451 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
452
453 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
454 type kcsan_atomic_readandclear_##name(volatile type *ptr) \
455 { \
456 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
457 __RET_ADDR); \
458 return (atomic_readandclear_##name(ptr)); \
459 }
460
461 #define _CSAN_ATOMIC_FUNC_SET(name, type) \
462 void kcsan_atomic_set_##name(volatile type *ptr, type val) \
463 { \
464 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
465 __RET_ADDR); \
466 atomic_set_##name(ptr, val); \
467 }
468
469 #define CSAN_ATOMIC_FUNC_SET(name, type) \
470 _CSAN_ATOMIC_FUNC_SET(name, type) \
471 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
472 _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
473
474 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
475 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
476 { \
477 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
478 __RET_ADDR); \
479 atomic_subtract_##name(ptr, val); \
480 }
481
482 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
483 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
484 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
485 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
486
487 #define _CSAN_ATOMIC_FUNC_STORE(name, type) \
488 void kcsan_atomic_store_##name(volatile type *ptr, type val) \
489 { \
490 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
491 __RET_ADDR); \
492 atomic_store_##name(ptr, val); \
493 }
494
495 #define CSAN_ATOMIC_FUNC_STORE(name, type) \
496 _CSAN_ATOMIC_FUNC_STORE(name, type) \
497 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
498
499 #define CSAN_ATOMIC_FUNC_SWAP(name, type) \
500 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
501 { \
502 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
503 __RET_ADDR); \
504 return(atomic_swap_##name(ptr, val)); \
505 }
506
507 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
508 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
509 { \
510 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
511 __RET_ADDR); \
512 return(atomic_testandclear_##name(ptr, val)); \
513 }
514
515 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
516 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
517 { \
518 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
519 __RET_ADDR); \
520 return (atomic_testandset_##name(ptr, val)); \
521 }
522
523 _CSAN_ATOMIC_FUNC_LOAD(bool, bool)
524 _CSAN_ATOMIC_FUNC_STORE(bool, bool)
525
526 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
527 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
528 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
529 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
530 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
531 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
532 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
533 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
534 #if 0
535 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
536 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
537 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
538 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
539 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
540 #endif
541
542 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
543 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
544 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
545 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
546 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
547 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
548 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
549 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
550 #if 0
551 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
552 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
553 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
554 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
555 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
556 #endif
557
558 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
559 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
560 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
561 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
562 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
563 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
564 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
565 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
566 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
567 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
568 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
569 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
570 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
571
572 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
573 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
574 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
575 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
576 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
577 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
578 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
579 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
580 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
581 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
582 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
583 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
584 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
585
586 CSAN_ATOMIC_FUNC_ADD(char, uint8_t)
587 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t)
588 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t)
589 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t)
590 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t)
591 CSAN_ATOMIC_FUNC_SET(char, uint8_t)
592 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t)
593 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t)
594 #if 0
595 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t)
596 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t)
597 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t)
598 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t)
599 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t)
600 #endif
601
602 CSAN_ATOMIC_FUNC_ADD(short, uint16_t)
603 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t)
604 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t)
605 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t)
606 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t)
607 CSAN_ATOMIC_FUNC_SET(short, uint16_t)
608 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t)
609 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t)
610 #if 0
611 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t)
612 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t)
613 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t)
614 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t)
615 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t)
616 #endif
617
618 CSAN_ATOMIC_FUNC_ADD(int, u_int)
619 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
620 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
621 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
622 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
623 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
624 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
625 CSAN_ATOMIC_FUNC_SET(int, u_int)
626 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
627 CSAN_ATOMIC_FUNC_STORE(int, u_int)
628 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
629 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
630 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
631
632 CSAN_ATOMIC_FUNC_ADD(long, u_long)
633 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
634 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
635 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
636 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
637 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
638 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
639 CSAN_ATOMIC_FUNC_SET(long, u_long)
640 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
641 CSAN_ATOMIC_FUNC_STORE(long, u_long)
642 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
643 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
644 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
645 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
646
647 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
648 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
649 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
650 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
651 #if !defined(__amd64__)
652 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
653 #endif
654 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
655 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
656 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
657 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
658 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
659 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
660 #if 0
661 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
662 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
663 #endif
664
665 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
666 void kcsan_atomic_thread_fence_##name(void) \
667 { \
668 atomic_thread_fence_##name(); \
669 }
670
671 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
672 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
673 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
674 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
675
676 void
677 kcsan_atomic_interrupt_fence(void)
678 {
679 atomic_interrupt_fence();
680 }
681
682 /* -------------------------------------------------------------------------- */
683
684 #include <sys/bus.h>
685 #include <machine/bus.h>
686 #include <sys/bus_san.h>
687
688 int
689 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
690 int flags, bus_space_handle_t *handlep)
691 {
692
693 return (bus_space_map(tag, hnd, size, flags, handlep));
694 }
695
696 void
697 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
698 bus_size_t size)
699 {
700
701 bus_space_unmap(tag, hnd, size);
702 }
703
704 int
705 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
706 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
707 {
708
709 return (bus_space_subregion(tag, hnd, offset, size, handlep));
710 }
711
712 #if !defined(__amd64__)
713 int
714 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
715 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
716 bus_size_t boundary, int flags, bus_addr_t *addrp,
717 bus_space_handle_t *handlep)
718 {
719
720 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
721 boundary, flags, addrp, handlep));
722 }
723 #endif
724
725 void
726 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
727 bus_size_t size)
728 {
729
730 bus_space_free(tag, hnd, size);
731 }
732
733 void
734 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
735 bus_size_t offset, bus_size_t size, int flags)
736 {
737
738 bus_space_barrier(tag, hnd, offset, size, flags);
739 }
740
741 #define CSAN_BUS_READ_FUNC(func, width, type) \
742 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
743 bus_space_handle_t hnd, bus_size_t offset) \
744 { \
745 return (bus_space_read##func##_##width(tag, hnd, \
746 offset)); \
747 } \
748
749 #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
750 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
751 bus_space_handle_t hnd, bus_size_t size, type *buf, \
752 bus_size_t count) \
753 { \
754 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
755 false, false, __RET_ADDR); \
756 bus_space_read_##func##_##width(tag, hnd, size, buf, \
757 count); \
758 }
759
760 CSAN_BUS_READ_FUNC(, 1, uint8_t)
761 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
762 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
763 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
764 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
765 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
766
767 CSAN_BUS_READ_FUNC(, 2, uint16_t)
768 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
769 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
770 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
771 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
772 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
773
774 CSAN_BUS_READ_FUNC(, 4, uint32_t)
775 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
776 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
777 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
778 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
779 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
780
781 CSAN_BUS_READ_FUNC(, 8, uint64_t)
782 #if defined(__aarch64__)
783 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
784 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
785 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
786 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
787 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
788 #endif
789
790 #define CSAN_BUS_WRITE_FUNC(func, width, type) \
791 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
792 bus_space_handle_t hnd, bus_size_t offset, type value) \
793 { \
794 bus_space_write##func##_##width(tag, hnd, offset, value); \
795 } \
796
797 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
798 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
799 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
800 bus_size_t count) \
801 { \
802 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
803 true, false, __RET_ADDR); \
804 bus_space_write_##func##_##width(tag, hnd, size, buf, \
805 count); \
806 }
807
808 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
809 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
810 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
811 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
812 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
813 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
814
815 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
816 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
817 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
818 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
819 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
820 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
821
822 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
823 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
824 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
825 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
826 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
827 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
828
829 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
830 #if defined(__aarch64__)
831 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
832 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
833 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
834 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
835 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
836 #endif
837
838 #define CSAN_BUS_SET_FUNC(func, width, type) \
839 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
840 bus_space_handle_t hnd, bus_size_t offset, type value, \
841 bus_size_t count) \
842 { \
843 bus_space_set_##func##_##width(tag, hnd, offset, value, \
844 count); \
845 }
846
847 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
848 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
849 #if !defined(__aarch64__)
850 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
851 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
852 #endif
853
854 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
855 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
856 #if !defined(__aarch64__)
857 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
858 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
859 #endif
860
861 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
862 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
863 #if !defined(__aarch64__)
864 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
865 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
866 #endif
867
868 #if !defined(__amd64__)
869 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
870 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
871 #if !defined(__aarch64__)
872 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
873 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
874 #endif
875 #endif
876
877 #define CSAN_BUS_PEEK_FUNC(width, type) \
878 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \
879 bus_space_handle_t hnd, bus_size_t offset, type *value) \
880 { \
881 kcsan_access((uintptr_t)value, sizeof(type), true, false, \
882 __RET_ADDR); \
883 return (bus_space_peek_##width(tag, hnd, offset, value)); \
884 }
885
886 CSAN_BUS_PEEK_FUNC(1, uint8_t)
887 CSAN_BUS_PEEK_FUNC(2, uint16_t)
888 CSAN_BUS_PEEK_FUNC(4, uint32_t)
889 #if !defined(__i386__)
890 CSAN_BUS_PEEK_FUNC(8, uint64_t)
891 #endif
892
893 #define CSAN_BUS_POKE_FUNC(width, type) \
894 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \
895 bus_space_handle_t hnd, bus_size_t offset, type value) \
896 { \
897 return (bus_space_poke_##width(tag, hnd, offset, value)); \
898 }
899
900 CSAN_BUS_POKE_FUNC(1, uint8_t)
901 CSAN_BUS_POKE_FUNC(2, uint16_t)
902 CSAN_BUS_POKE_FUNC(4, uint32_t)
903 #if !defined(__i386__)
904 CSAN_BUS_POKE_FUNC(8, uint64_t)
905 #endif
Cache object: 7a30679d2b3ebacc481d7c4c5bb28013
|