Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_csan.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * Copyright (c) 2019 Andrew Turner 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Maxime Villard. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define SAN_RUNTIME 34 35 #include "opt_ddb.h" 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include <sys/param.h> 41 #include <sys/kernel.h> 42 #include <sys/bus.h> 43 #include <sys/conf.h> 44 #include <sys/cpu.h> 45 #include <sys/csan.h> 46 #include <sys/proc.h> 47 #include <sys/smp.h> 48 #include <sys/systm.h> 49 50 #include <ddb/ddb.h> 51 #include <ddb/db_sym.h> 52 53 #ifdef KCSAN_PANIC 54 #define REPORT panic 55 #else 56 #define REPORT printf 57 #endif 58 59 typedef struct { 60 uintptr_t addr; 61 uint32_t size; 62 bool write:1; 63 bool atomic:1; 64 uintptr_t pc; 65 } csan_cell_t; 66 67 typedef struct { 68 bool inited; 69 uint32_t cnt; 70 csan_cell_t cell; 71 } csan_cpu_t; 72 73 static csan_cpu_t kcsan_cpus[MAXCPU]; 74 static bool kcsan_enabled __read_mostly; 75 76 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 77 78 #define KCSAN_NACCESSES 1024 79 #define KCSAN_DELAY 10 /* 10 microseconds */ 80 81 /* -------------------------------------------------------------------------- */ 82 83 /* The MD code. */ 84 #include <machine/csan.h> 85 86 /* -------------------------------------------------------------------------- */ 87 88 static void 89 kcsan_enable(void *dummy __unused) 90 { 91 92 printf("Enabling KCSCAN, expect reduced performance.\n"); 93 kcsan_enabled = true; 94 } 95 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); 96 97 void 98 kcsan_cpu_init(u_int cpu) 99 { 100 kcsan_cpus[cpu].inited = true; 101 } 102 103 /* -------------------------------------------------------------------------- */ 104 105 static inline void 106 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) 107 { 108 const char *newsym, *oldsym; 109 #ifdef DDB 110 c_db_sym_t sym; 111 db_expr_t offset; 112 113 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); 114 db_symbol_values(sym, &newsym, NULL); 115 116 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); 117 db_symbol_values(sym, &oldsym, NULL); 118 #else 119 newsym = ""; 120 oldsym = ""; 121 #endif 122 REPORT("CSan: Racy Access " 123 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " 124 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", 125 newcpu, 126 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 127 (void *)new->addr, new->size, (void *)new->pc, newsym, 128 oldcpu, 129 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 130 (void *)old->addr, old->size, (void *)old->pc, oldsym); 131 kcsan_md_unwind(); 132 } 133 134 static inline bool 135 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 136 { 137 if (new->write && !new->atomic) 138 return false; 139 if (old->write && !old->atomic) 140 return false; 141 return true; 142 } 143 144 static inline void 145 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 146 { 147 csan_cell_t old, new; 148 csan_cpu_t *cpu; 149 uint64_t intr; 150 size_t i; 151 152 if (__predict_false(!kcsan_enabled)) 153 return; 154 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) 155 return; 156 if (KERNEL_PANICKED()) 157 return; 158 159 new.addr = addr; 160 new.size = size; 161 new.write = write; 162 new.atomic = atomic; 163 new.pc = pc; 164 165 CPU_FOREACH(i) { 166 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 167 168 if (old.addr + old.size <= new.addr) 169 continue; 170 if (new.addr + new.size <= old.addr) 171 continue; 172 if (__predict_true(!old.write && !new.write)) 173 continue; 174 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 175 continue; 176 177 kcsan_report(&new, PCPU_GET(cpuid), &old, i); 178 break; 179 } 180 181 if (__predict_false(!kcsan_md_is_avail())) 182 return; 183 184 kcsan_md_disable_intrs(&intr); 185 186 cpu = &kcsan_cpus[PCPU_GET(cpuid)]; 187 if (__predict_false(!cpu->inited)) 188 goto out; 189 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 190 if (__predict_true(cpu->cnt != 0)) 191 goto out; 192 193 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 194 kcsan_md_delay(KCSAN_DELAY); 195 __builtin_memset(&cpu->cell, 0, sizeof(new)); 196 197 out: 198 kcsan_md_enable_intrs(&intr); 199 } 200 201 #define CSAN_READ(size) \ 202 void __tsan_read##size(uintptr_t); \ 203 void __tsan_read##size(uintptr_t addr) \ 204 { \ 205 kcsan_access(addr, size, false, false, __RET_ADDR); \ 206 } \ 207 void __tsan_unaligned_read##size(uintptr_t); \ 208 void __tsan_unaligned_read##size(uintptr_t addr) \ 209 { \ 210 kcsan_access(addr, size, false, false, __RET_ADDR); \ 211 } 212 213 CSAN_READ(1) 214 CSAN_READ(2) 215 CSAN_READ(4) 216 CSAN_READ(8) 217 CSAN_READ(16) 218 219 #define CSAN_WRITE(size) \ 220 void __tsan_write##size(uintptr_t); \ 221 void __tsan_write##size(uintptr_t addr) \ 222 { \ 223 kcsan_access(addr, size, true, false, __RET_ADDR); \ 224 } \ 225 void __tsan_unaligned_write##size(uintptr_t); \ 226 void __tsan_unaligned_write##size(uintptr_t addr) \ 227 { \ 228 kcsan_access(addr, size, true, false, __RET_ADDR); \ 229 } 230 231 CSAN_WRITE(1) 232 CSAN_WRITE(2) 233 CSAN_WRITE(4) 234 CSAN_WRITE(8) 235 CSAN_WRITE(16) 236 237 void __tsan_read_range(uintptr_t, size_t); 238 void __tsan_write_range(uintptr_t, size_t); 239 240 void 241 __tsan_read_range(uintptr_t addr, size_t size) 242 { 243 kcsan_access(addr, size, false, false, __RET_ADDR); 244 } 245 246 void 247 __tsan_write_range(uintptr_t addr, size_t size) 248 { 249 kcsan_access(addr, size, true, false, __RET_ADDR); 250 } 251 252 void __tsan_init(void); 253 void __tsan_func_entry(void *); 254 void __tsan_func_exit(void); 255 256 void 257 __tsan_init(void) 258 { 259 } 260 261 void 262 __tsan_func_entry(void *call_pc) 263 { 264 } 265 266 void 267 __tsan_func_exit(void) 268 { 269 } 270 271 /* -------------------------------------------------------------------------- */ 272 273 void * 274 kcsan_memcpy(void *dst, const void *src, size_t len) 275 { 276 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 277 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 278 return __builtin_memcpy(dst, src, len); 279 } 280 281 int 282 kcsan_memcmp(const void *b1, const void *b2, size_t len) 283 { 284 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 285 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 286 return __builtin_memcmp(b1, b2, len); 287 } 288 289 void * 290 kcsan_memset(void *b, int c, size_t len) 291 { 292 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 293 return __builtin_memset(b, c, len); 294 } 295 296 void * 297 kcsan_memmove(void *dst, const void *src, size_t len) 298 { 299 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 300 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 301 return __builtin_memmove(dst, src, len); 302 } 303 304 char * 305 kcsan_strcpy(char *dst, const char *src) 306 { 307 char *save = dst; 308 309 while (1) { 310 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 311 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 312 *dst = *src; 313 if (*src == '\0') 314 break; 315 src++, dst++; 316 } 317 318 return save; 319 } 320 321 int 322 kcsan_strcmp(const char *s1, const char *s2) 323 { 324 while (1) { 325 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 326 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 327 if (*s1 != *s2) 328 break; 329 if (*s1 == '\0') 330 return 0; 331 s1++, s2++; 332 } 333 334 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 335 } 336 337 size_t 338 kcsan_strlen(const char *str) 339 { 340 const char *s; 341 342 s = str; 343 while (1) { 344 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 345 if (*s == '\0') 346 break; 347 s++; 348 } 349 350 return (s - str); 351 } 352 353 int 354 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 355 { 356 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 357 return copyin(uaddr, kaddr, len); 358 } 359 360 int 361 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 362 { 363 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 364 return copyinstr(uaddr, kaddr, len, done); 365 } 366 367 int 368 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 369 { 370 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 371 return copyout(kaddr, uaddr, len); 372 } 373 374 /* -------------------------------------------------------------------------- */ 375 376 #include <machine/atomic.h> 377 #include <sys/atomic_san.h> 378 379 #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ 380 void kcsan_atomic_add_##name(volatile type *ptr, type val) \ 381 { \ 382 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 383 __RET_ADDR); \ 384 atomic_add_##name(ptr, val); \ 385 } 386 387 #define CSAN_ATOMIC_FUNC_ADD(name, type) \ 388 _CSAN_ATOMIC_FUNC_ADD(name, type) \ 389 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 390 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) 391 392 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 393 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ 394 { \ 395 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 396 __RET_ADDR); \ 397 atomic_clear_##name(ptr, val); \ 398 } 399 400 #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 401 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 402 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 403 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 404 405 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 406 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ 407 type val2) \ 408 { \ 409 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 410 __RET_ADDR); \ 411 return (atomic_cmpset_##name(ptr, val1, val2)); \ 412 } 413 414 #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 415 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 416 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 417 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 418 419 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 420 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ 421 type val2) \ 422 { \ 423 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 424 __RET_ADDR); \ 425 return (atomic_fcmpset_##name(ptr, val1, val2)); \ 426 } 427 428 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 429 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 430 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 431 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 432 433 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 434 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 435 { \ 436 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 437 __RET_ADDR); \ 438 return (atomic_fetchadd_##name(ptr, val)); \ 439 } 440 441 #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 442 type kcsan_atomic_load_##name(volatile type *ptr) \ 443 { \ 444 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ 445 __RET_ADDR); \ 446 return (atomic_load_##name(ptr)); \ 447 } 448 449 #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ 450 _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 451 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ 452 453 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 454 type kcsan_atomic_readandclear_##name(volatile type *ptr) \ 455 { \ 456 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 457 __RET_ADDR); \ 458 return (atomic_readandclear_##name(ptr)); \ 459 } 460 461 #define _CSAN_ATOMIC_FUNC_SET(name, type) \ 462 void kcsan_atomic_set_##name(volatile type *ptr, type val) \ 463 { \ 464 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 465 __RET_ADDR); \ 466 atomic_set_##name(ptr, val); \ 467 } 468 469 #define CSAN_ATOMIC_FUNC_SET(name, type) \ 470 _CSAN_ATOMIC_FUNC_SET(name, type) \ 471 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 472 _CSAN_ATOMIC_FUNC_SET(rel_##name, type) 473 474 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 475 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ 476 { \ 477 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 478 __RET_ADDR); \ 479 atomic_subtract_##name(ptr, val); \ 480 } 481 482 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 483 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 484 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 485 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 486 487 #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ 488 void kcsan_atomic_store_##name(volatile type *ptr, type val) \ 489 { \ 490 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 491 __RET_ADDR); \ 492 atomic_store_##name(ptr, val); \ 493 } 494 495 #define CSAN_ATOMIC_FUNC_STORE(name, type) \ 496 _CSAN_ATOMIC_FUNC_STORE(name, type) \ 497 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) 498 499 #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ 500 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ 501 { \ 502 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 503 __RET_ADDR); \ 504 return(atomic_swap_##name(ptr, val)); \ 505 } 506 507 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 508 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ 509 { \ 510 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 511 __RET_ADDR); \ 512 return(atomic_testandclear_##name(ptr, val)); \ 513 } 514 515 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 516 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ 517 { \ 518 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 519 __RET_ADDR); \ 520 return (atomic_testandset_##name(ptr, val)); \ 521 } 522 523 _CSAN_ATOMIC_FUNC_LOAD(bool, bool) 524 _CSAN_ATOMIC_FUNC_STORE(bool, bool) 525 526 CSAN_ATOMIC_FUNC_ADD(8, uint8_t) 527 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) 528 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) 529 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) 530 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) 531 CSAN_ATOMIC_FUNC_SET(8, uint8_t) 532 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) 533 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) 534 #if 0 535 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) 536 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) 537 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) 538 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) 539 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) 540 #endif 541 542 CSAN_ATOMIC_FUNC_ADD(16, uint16_t) 543 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) 544 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) 545 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) 546 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) 547 CSAN_ATOMIC_FUNC_SET(16, uint16_t) 548 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) 549 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) 550 #if 0 551 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) 552 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) 553 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) 554 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) 555 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) 556 #endif 557 558 CSAN_ATOMIC_FUNC_ADD(32, uint32_t) 559 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) 560 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) 561 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) 562 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) 563 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) 564 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) 565 CSAN_ATOMIC_FUNC_SET(32, uint32_t) 566 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) 567 CSAN_ATOMIC_FUNC_STORE(32, uint32_t) 568 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) 569 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) 570 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) 571 572 CSAN_ATOMIC_FUNC_ADD(64, uint64_t) 573 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) 574 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) 575 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) 576 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) 577 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) 578 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) 579 CSAN_ATOMIC_FUNC_SET(64, uint64_t) 580 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) 581 CSAN_ATOMIC_FUNC_STORE(64, uint64_t) 582 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) 583 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) 584 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) 585 586 CSAN_ATOMIC_FUNC_ADD(char, uint8_t) 587 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) 588 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) 589 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) 590 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) 591 CSAN_ATOMIC_FUNC_SET(char, uint8_t) 592 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) 593 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) 594 #if 0 595 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) 596 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) 597 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) 598 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) 599 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) 600 #endif 601 602 CSAN_ATOMIC_FUNC_ADD(short, uint16_t) 603 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) 604 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) 605 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) 606 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) 607 CSAN_ATOMIC_FUNC_SET(short, uint16_t) 608 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) 609 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) 610 #if 0 611 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) 612 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) 613 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) 614 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) 615 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) 616 #endif 617 618 CSAN_ATOMIC_FUNC_ADD(int, u_int) 619 CSAN_ATOMIC_FUNC_CLEAR(int, u_int) 620 CSAN_ATOMIC_FUNC_CMPSET(int, u_int) 621 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) 622 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) 623 CSAN_ATOMIC_FUNC_LOAD(int, u_int) 624 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) 625 CSAN_ATOMIC_FUNC_SET(int, u_int) 626 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) 627 CSAN_ATOMIC_FUNC_STORE(int, u_int) 628 CSAN_ATOMIC_FUNC_SWAP(int, u_int) 629 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) 630 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) 631 632 CSAN_ATOMIC_FUNC_ADD(long, u_long) 633 CSAN_ATOMIC_FUNC_CLEAR(long, u_long) 634 CSAN_ATOMIC_FUNC_CMPSET(long, u_long) 635 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) 636 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) 637 CSAN_ATOMIC_FUNC_LOAD(long, u_long) 638 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) 639 CSAN_ATOMIC_FUNC_SET(long, u_long) 640 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) 641 CSAN_ATOMIC_FUNC_STORE(long, u_long) 642 CSAN_ATOMIC_FUNC_SWAP(long, u_long) 643 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) 644 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) 645 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) 646 647 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) 648 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) 649 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) 650 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) 651 #if !defined(__amd64__) 652 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) 653 #endif 654 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) 655 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) 656 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) 657 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) 658 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) 659 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) 660 #if 0 661 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) 662 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) 663 #endif 664 665 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 666 void kcsan_atomic_thread_fence_##name(void) \ 667 { \ 668 atomic_thread_fence_##name(); \ 669 } 670 671 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) 672 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) 673 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) 674 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) 675 676 void 677 kcsan_atomic_interrupt_fence(void) 678 { 679 atomic_interrupt_fence(); 680 } 681 682 /* -------------------------------------------------------------------------- */ 683 684 #include <sys/bus.h> 685 #include <machine/bus.h> 686 #include <sys/bus_san.h> 687 688 int 689 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 690 int flags, bus_space_handle_t *handlep) 691 { 692 693 return (bus_space_map(tag, hnd, size, flags, handlep)); 694 } 695 696 void 697 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 698 bus_size_t size) 699 { 700 701 bus_space_unmap(tag, hnd, size); 702 } 703 704 int 705 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 706 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 707 { 708 709 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 710 } 711 712 #if !defined(__amd64__) 713 int 714 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, 715 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, 716 bus_size_t boundary, int flags, bus_addr_t *addrp, 717 bus_space_handle_t *handlep) 718 { 719 720 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, 721 boundary, flags, addrp, handlep)); 722 } 723 #endif 724 725 void 726 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 727 bus_size_t size) 728 { 729 730 bus_space_free(tag, hnd, size); 731 } 732 733 void 734 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 735 bus_size_t offset, bus_size_t size, int flags) 736 { 737 738 bus_space_barrier(tag, hnd, offset, size, flags); 739 } 740 741 #define CSAN_BUS_READ_FUNC(func, width, type) \ 742 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 743 bus_space_handle_t hnd, bus_size_t offset) \ 744 { \ 745 return (bus_space_read##func##_##width(tag, hnd, \ 746 offset)); \ 747 } \ 748 749 #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ 750 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 751 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 752 bus_size_t count) \ 753 { \ 754 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 755 false, false, __RET_ADDR); \ 756 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 757 count); \ 758 } 759 760 CSAN_BUS_READ_FUNC(, 1, uint8_t) 761 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 762 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 763 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 764 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 765 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 766 767 CSAN_BUS_READ_FUNC(, 2, uint16_t) 768 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 769 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 770 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 771 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 772 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 773 774 CSAN_BUS_READ_FUNC(, 4, uint32_t) 775 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 776 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 777 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 778 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 779 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 780 781 CSAN_BUS_READ_FUNC(, 8, uint64_t) 782 #if defined(__aarch64__) 783 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) 784 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) 785 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) 786 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) 787 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) 788 #endif 789 790 #define CSAN_BUS_WRITE_FUNC(func, width, type) \ 791 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 792 bus_space_handle_t hnd, bus_size_t offset, type value) \ 793 { \ 794 bus_space_write##func##_##width(tag, hnd, offset, value); \ 795 } \ 796 797 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 798 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ 799 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 800 bus_size_t count) \ 801 { \ 802 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 803 true, false, __RET_ADDR); \ 804 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 805 count); \ 806 } 807 808 CSAN_BUS_WRITE_FUNC(, 1, uint8_t) 809 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 810 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 811 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 812 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 813 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 814 815 CSAN_BUS_WRITE_FUNC(, 2, uint16_t) 816 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 817 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 818 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 819 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 820 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 821 822 CSAN_BUS_WRITE_FUNC(, 4, uint32_t) 823 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 824 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 825 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 826 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 827 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 828 829 CSAN_BUS_WRITE_FUNC(, 8, uint64_t) 830 #if defined(__aarch64__) 831 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) 832 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) 833 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) 834 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) 835 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) 836 #endif 837 838 #define CSAN_BUS_SET_FUNC(func, width, type) \ 839 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 840 bus_space_handle_t hnd, bus_size_t offset, type value, \ 841 bus_size_t count) \ 842 { \ 843 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 844 count); \ 845 } 846 847 CSAN_BUS_SET_FUNC(multi, 1, uint8_t) 848 CSAN_BUS_SET_FUNC(region, 1, uint8_t) 849 #if !defined(__aarch64__) 850 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 851 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 852 #endif 853 854 CSAN_BUS_SET_FUNC(multi, 2, uint16_t) 855 CSAN_BUS_SET_FUNC(region, 2, uint16_t) 856 #if !defined(__aarch64__) 857 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 858 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 859 #endif 860 861 CSAN_BUS_SET_FUNC(multi, 4, uint32_t) 862 CSAN_BUS_SET_FUNC(region, 4, uint32_t) 863 #if !defined(__aarch64__) 864 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 865 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 866 #endif 867 868 #if !defined(__amd64__) 869 CSAN_BUS_SET_FUNC(multi, 8, uint64_t) 870 CSAN_BUS_SET_FUNC(region, 8, uint64_t) 871 #if !defined(__aarch64__) 872 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) 873 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) 874 #endif 875 #endif 876 877 #define CSAN_BUS_PEEK_FUNC(width, type) \ 878 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ 879 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 880 { \ 881 kcsan_access((uintptr_t)value, sizeof(type), true, false, \ 882 __RET_ADDR); \ 883 return (bus_space_peek_##width(tag, hnd, offset, value)); \ 884 } 885 886 CSAN_BUS_PEEK_FUNC(1, uint8_t) 887 CSAN_BUS_PEEK_FUNC(2, uint16_t) 888 CSAN_BUS_PEEK_FUNC(4, uint32_t) 889 #if !defined(__i386__) 890 CSAN_BUS_PEEK_FUNC(8, uint64_t) 891 #endif 892 893 #define CSAN_BUS_POKE_FUNC(width, type) \ 894 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ 895 bus_space_handle_t hnd, bus_size_t offset, type value) \ 896 { \ 897 return (bus_space_poke_##width(tag, hnd, offset, value)); \ 898 } 899 900 CSAN_BUS_POKE_FUNC(1, uint8_t) 901 CSAN_BUS_POKE_FUNC(2, uint16_t) 902 CSAN_BUS_POKE_FUNC(4, uint32_t) 903 #if !defined(__i386__) 904 CSAN_BUS_POKE_FUNC(8, uint64_t) 905 #endif
Cache object: 7a30679d2b3ebacc481d7c4c5bb28013
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]