Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_kcov.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. 5 * Copyright (C) 2018, 2019 Andrew Turner 6 * 7 * This software was developed by Mitchell Horne under sponsorship of 8 * the FreeBSD Foundation. 9 * 10 * This software was developed by SRI International and the University of 11 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 12 * ("CTSRD"), as part of the DARPA CRASH research programme. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * $FreeBSD$ 36 */ 37 38 /* Interceptors are required for KMSAN. */ 39 #if defined(KASAN) || defined(KCSAN) 40 #define SAN_RUNTIME 41 #endif 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/conf.h> 49 #include <sys/eventhandler.h> 50 #include <sys/kcov.h> 51 #include <sys/kernel.h> 52 #include <sys/limits.h> 53 #include <sys/lock.h> 54 #include <sys/malloc.h> 55 #include <sys/mman.h> 56 #include <sys/mutex.h> 57 #include <sys/proc.h> 58 #include <sys/rwlock.h> 59 #include <sys/sysctl.h> 60 61 #include <vm/vm.h> 62 #include <vm/pmap.h> 63 #include <vm/vm_extern.h> 64 #include <vm/vm_object.h> 65 #include <vm/vm_page.h> 66 #include <vm/vm_pager.h> 67 #include <vm/vm_param.h> 68 69 MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); 70 71 #define KCOV_ELEMENT_SIZE sizeof(uint64_t) 72 73 /* 74 * To know what the code can safely perform at any point in time we use a 75 * state machine. In the normal case the state transitions are: 76 * 77 * OPEN -> READY -> RUNNING -> DYING 78 * | | ^ | ^ ^ 79 * | | +--------+ | | 80 * | +-------------------+ | 81 * +-----------------------------+ 82 * 83 * The states are: 84 * OPEN: The kcov fd has been opened, but no buffer is available to store 85 * coverage data. 86 * READY: The buffer to store coverage data has been allocated. Userspace 87 * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When 88 * this has been set the buffer can be written to by the kernel, 89 * and mmaped by userspace. 90 * RUNNING: The coverage probes are able to store coverage data in the buffer. 91 * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state 92 * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to 93 * return to the READY state to allow tracing to be reused, or by 94 * closing the kcov fd to enter the DYING state. 95 * DYING: The fd has been closed. All states can enter into this state when 96 * userspace closes the kcov fd. 97 * 98 * We need to be careful when moving into and out of the RUNNING state. As 99 * an interrupt may happen while this is happening the ordering of memory 100 * operations is important so struct kcov_info is valid for the tracing 101 * functions. 102 * 103 * When moving into the RUNNING state prior stores to struct kcov_info need 104 * to be observed before the state is set. This allows for interrupts that 105 * may call into one of the coverage functions to fire at any point while 106 * being enabled and see a consistent struct kcov_info. 107 * 108 * When moving out of the RUNNING state any later stores to struct kcov_info 109 * need to be observed after the state is set. As with entering this is to 110 * present a consistent struct kcov_info to interrupts. 111 */ 112 typedef enum { 113 KCOV_STATE_INVALID, 114 KCOV_STATE_OPEN, /* The device is open, but with no buffer */ 115 KCOV_STATE_READY, /* The buffer has been allocated */ 116 KCOV_STATE_RUNNING, /* Recording trace data */ 117 KCOV_STATE_DYING, /* The fd was closed */ 118 } kcov_state_t; 119 120 /* 121 * (l) Set while holding the kcov_lock mutex and not in the RUNNING state. 122 * (o) Only set once while in the OPEN state. Cleaned up while in the DYING 123 * state, and with no thread associated with the struct kcov_info. 124 * (s) Set atomically to enter or exit the RUNNING state, non-atomically 125 * otherwise. See above for a description of the other constraints while 126 * moving into or out of the RUNNING state. 127 */ 128 struct kcov_info { 129 struct thread *thread; /* (l) */ 130 vm_object_t bufobj; /* (o) */ 131 vm_offset_t kvaddr; /* (o) */ 132 size_t entries; /* (o) */ 133 size_t bufsize; /* (o) */ 134 kcov_state_t state; /* (s) */ 135 int mode; /* (l) */ 136 }; 137 138 /* Prototypes */ 139 static d_open_t kcov_open; 140 static d_close_t kcov_close; 141 static d_mmap_single_t kcov_mmap_single; 142 static d_ioctl_t kcov_ioctl; 143 144 static int kcov_alloc(struct kcov_info *info, size_t entries); 145 static void kcov_free(struct kcov_info *info); 146 static void kcov_init(const void *unused); 147 148 static struct cdevsw kcov_cdevsw = { 149 .d_version = D_VERSION, 150 .d_open = kcov_open, 151 .d_close = kcov_close, 152 .d_mmap_single = kcov_mmap_single, 153 .d_ioctl = kcov_ioctl, 154 .d_name = "kcov", 155 }; 156 157 SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 158 "Kernel coverage"); 159 160 static u_int kcov_max_entries = KCOV_MAXENTRIES; 161 SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW, 162 &kcov_max_entries, 0, 163 "Maximum number of entries in the kcov buffer"); 164 165 static struct mtx kcov_lock; 166 static int active_count; 167 168 static struct kcov_info * __nosanitizeaddress __nosanitizememory 169 get_kinfo(struct thread *td) 170 { 171 struct kcov_info *info; 172 173 /* We might have a NULL thread when releasing the secondary CPUs */ 174 if (td == NULL) 175 return (NULL); 176 177 /* 178 * We are in an interrupt, stop tracing as it is not explicitly 179 * part of a syscall. 180 */ 181 if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL) 182 return (NULL); 183 184 /* 185 * If info is NULL or the state is not running we are not tracing. 186 */ 187 info = td->td_kcov_info; 188 if (info == NULL || 189 atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING) 190 return (NULL); 191 192 return (info); 193 } 194 195 static void __nosanitizeaddress __nosanitizememory 196 trace_pc(uintptr_t ret) 197 { 198 struct thread *td; 199 struct kcov_info *info; 200 uint64_t *buf, index; 201 202 td = curthread; 203 info = get_kinfo(td); 204 if (info == NULL) 205 return; 206 207 /* 208 * Check we are in the PC-trace mode. 209 */ 210 if (info->mode != KCOV_MODE_TRACE_PC) 211 return; 212 213 KASSERT(info->kvaddr != 0, ("%s: NULL buf while running", __func__)); 214 215 buf = (uint64_t *)info->kvaddr; 216 217 /* The first entry of the buffer holds the index */ 218 index = buf[0]; 219 if (index + 2 > info->entries) 220 return; 221 222 buf[index + 1] = ret; 223 buf[0] = index + 1; 224 } 225 226 static bool __nosanitizeaddress __nosanitizememory 227 trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret) 228 { 229 struct thread *td; 230 struct kcov_info *info; 231 uint64_t *buf, index; 232 233 td = curthread; 234 info = get_kinfo(td); 235 if (info == NULL) 236 return (false); 237 238 /* 239 * Check we are in the comparison-trace mode. 240 */ 241 if (info->mode != KCOV_MODE_TRACE_CMP) 242 return (false); 243 244 KASSERT(info->kvaddr != 0, ("%s: NULL buf while running", __func__)); 245 246 buf = (uint64_t *)info->kvaddr; 247 248 /* The first entry of the buffer holds the index */ 249 index = buf[0]; 250 251 /* Check we have space to store all elements */ 252 if (index * 4 + 4 + 1 > info->entries) 253 return (false); 254 255 while (1) { 256 buf[index * 4 + 1] = type; 257 buf[index * 4 + 2] = arg1; 258 buf[index * 4 + 3] = arg2; 259 buf[index * 4 + 4] = ret; 260 261 if (atomic_cmpset_64(&buf[0], index, index + 1)) 262 break; 263 buf[0] = index; 264 } 265 266 return (true); 267 } 268 269 /* 270 * The fd is being closed, cleanup everything we can. 271 */ 272 static void 273 kcov_mmap_cleanup(void *arg) 274 { 275 struct kcov_info *info = arg; 276 struct thread *thread; 277 278 mtx_lock_spin(&kcov_lock); 279 /* 280 * Move to KCOV_STATE_DYING to stop adding new entries. 281 * 282 * If the thread is running we need to wait until thread exit to 283 * clean up as it may currently be adding a new entry. If this is 284 * the case being in KCOV_STATE_DYING will signal that the buffer 285 * needs to be cleaned up. 286 */ 287 atomic_store_int(&info->state, KCOV_STATE_DYING); 288 atomic_thread_fence_seq_cst(); 289 thread = info->thread; 290 mtx_unlock_spin(&kcov_lock); 291 292 if (thread != NULL) 293 return; 294 295 /* 296 * We can safely clean up the info struct as it is in the 297 * KCOV_STATE_DYING state with no thread associated. 298 * 299 * The KCOV_STATE_DYING stops new threads from using it. 300 * The lack of a thread means nothing is currently using the buffers. 301 */ 302 kcov_free(info); 303 } 304 305 static int 306 kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 307 { 308 struct kcov_info *info; 309 int error; 310 311 info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK); 312 info->state = KCOV_STATE_OPEN; 313 info->thread = NULL; 314 info->mode = -1; 315 316 if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0) 317 kcov_mmap_cleanup(info); 318 319 return (error); 320 } 321 322 static int 323 kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 324 { 325 struct kcov_info *info; 326 int error; 327 328 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 329 return (error); 330 331 KASSERT(info != NULL, ("kcov_close with no kcov_info structure")); 332 333 /* Trying to close, but haven't disabled */ 334 if (info->state == KCOV_STATE_RUNNING) 335 return (EBUSY); 336 337 return (0); 338 } 339 340 static int 341 kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, 342 struct vm_object **object, int nprot) 343 { 344 struct kcov_info *info; 345 int error; 346 347 if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) != 348 (PROT_READ | PROT_WRITE)) 349 return (EINVAL); 350 351 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 352 return (error); 353 354 if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries) 355 return (EINVAL); 356 357 vm_object_reference(info->bufobj); 358 *offset = 0; 359 *object = info->bufobj; 360 return (0); 361 } 362 363 static int 364 kcov_alloc(struct kcov_info *info, size_t entries) 365 { 366 size_t n, pages; 367 vm_page_t m; 368 369 KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer")); 370 KASSERT(info->state == KCOV_STATE_OPEN, 371 ("kcov_alloc: Not in open state (%x)", info->state)); 372 373 if (entries < 2 || entries > kcov_max_entries) 374 return (EINVAL); 375 376 /* Align to page size so mmap can't access other kernel memory */ 377 info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE); 378 pages = info->bufsize / PAGE_SIZE; 379 380 if ((info->kvaddr = kva_alloc(info->bufsize)) == 0) 381 return (ENOMEM); 382 383 info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize, 384 PROT_READ | PROT_WRITE, 0, curthread->td_ucred); 385 386 VM_OBJECT_WLOCK(info->bufobj); 387 for (n = 0; n < pages; n++) { 388 m = vm_page_grab(info->bufobj, n, 389 VM_ALLOC_ZERO | VM_ALLOC_WIRED); 390 vm_page_valid(m); 391 vm_page_xunbusy(m); 392 pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1); 393 } 394 VM_OBJECT_WUNLOCK(info->bufobj); 395 396 info->entries = entries; 397 398 return (0); 399 } 400 401 static void 402 kcov_free(struct kcov_info *info) 403 { 404 vm_page_t m; 405 size_t i; 406 407 if (info->kvaddr != 0) { 408 pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE); 409 kva_free(info->kvaddr, info->bufsize); 410 } 411 if (info->bufobj != NULL) { 412 VM_OBJECT_WLOCK(info->bufobj); 413 m = vm_page_lookup(info->bufobj, 0); 414 for (i = 0; i < info->bufsize / PAGE_SIZE; i++) { 415 vm_page_unwire_noq(m); 416 m = vm_page_next(m); 417 } 418 VM_OBJECT_WUNLOCK(info->bufobj); 419 vm_object_deallocate(info->bufobj); 420 } 421 free(info, M_KCOV_INFO); 422 } 423 424 static int 425 kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused, 426 struct thread *td) 427 { 428 struct kcov_info *info; 429 int mode, error; 430 431 if ((error = devfs_get_cdevpriv((void **)&info)) != 0) 432 return (error); 433 434 if (cmd == KIOSETBUFSIZE) { 435 /* 436 * Set the size of the coverage buffer. Should be called 437 * before enabling coverage collection for that thread. 438 */ 439 if (info->state != KCOV_STATE_OPEN) { 440 return (EBUSY); 441 } 442 error = kcov_alloc(info, *(u_int *)data); 443 if (error == 0) 444 info->state = KCOV_STATE_READY; 445 return (error); 446 } 447 448 mtx_lock_spin(&kcov_lock); 449 switch (cmd) { 450 case KIOENABLE: 451 if (info->state != KCOV_STATE_READY) { 452 error = EBUSY; 453 break; 454 } 455 if (td->td_kcov_info != NULL) { 456 error = EINVAL; 457 break; 458 } 459 mode = *(int *)data; 460 if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) { 461 error = EINVAL; 462 break; 463 } 464 465 /* Lets hope nobody opens this 2 billion times */ 466 KASSERT(active_count < INT_MAX, 467 ("%s: Open too many times", __func__)); 468 active_count++; 469 if (active_count == 1) { 470 cov_register_pc(&trace_pc); 471 cov_register_cmp(&trace_cmp); 472 } 473 474 KASSERT(info->thread == NULL, 475 ("Enabling kcov when already enabled")); 476 info->thread = td; 477 info->mode = mode; 478 /* 479 * Ensure the mode has been set before starting coverage 480 * tracing. 481 */ 482 atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING); 483 td->td_kcov_info = info; 484 break; 485 case KIODISABLE: 486 /* Only the currently enabled thread may disable itself */ 487 if (info->state != KCOV_STATE_RUNNING || 488 info != td->td_kcov_info) { 489 error = EINVAL; 490 break; 491 } 492 KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); 493 active_count--; 494 if (active_count == 0) { 495 cov_unregister_pc(); 496 cov_unregister_cmp(); 497 } 498 499 td->td_kcov_info = NULL; 500 atomic_store_int(&info->state, KCOV_STATE_READY); 501 /* 502 * Ensure we have exited the READY state before clearing the 503 * rest of the info struct. 504 */ 505 atomic_thread_fence_rel(); 506 info->mode = -1; 507 info->thread = NULL; 508 break; 509 default: 510 error = EINVAL; 511 break; 512 } 513 mtx_unlock_spin(&kcov_lock); 514 515 return (error); 516 } 517 518 static void 519 kcov_thread_dtor(void *arg __unused, struct thread *td) 520 { 521 struct kcov_info *info; 522 523 info = td->td_kcov_info; 524 if (info == NULL) 525 return; 526 527 mtx_lock_spin(&kcov_lock); 528 KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); 529 active_count--; 530 if (active_count == 0) { 531 cov_unregister_pc(); 532 cov_unregister_cmp(); 533 } 534 td->td_kcov_info = NULL; 535 if (info->state != KCOV_STATE_DYING) { 536 /* 537 * The kcov file is still open. Mark it as unused and 538 * wait for it to be closed before cleaning up. 539 */ 540 atomic_store_int(&info->state, KCOV_STATE_READY); 541 atomic_thread_fence_seq_cst(); 542 /* This info struct is unused */ 543 info->thread = NULL; 544 mtx_unlock_spin(&kcov_lock); 545 return; 546 } 547 mtx_unlock_spin(&kcov_lock); 548 549 /* 550 * We can safely clean up the info struct as it is in the 551 * KCOV_STATE_DYING state where the info struct is associated with 552 * the current thread that's about to exit. 553 * 554 * The KCOV_STATE_DYING stops new threads from using it. 555 * It also stops the current thread from trying to use the info struct. 556 */ 557 kcov_free(info); 558 } 559 560 static void 561 kcov_init(const void *unused) 562 { 563 struct make_dev_args args; 564 struct cdev *dev; 565 566 mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN); 567 568 make_dev_args_init(&args); 569 args.mda_devsw = &kcov_cdevsw; 570 args.mda_uid = UID_ROOT; 571 args.mda_gid = GID_WHEEL; 572 args.mda_mode = 0600; 573 if (make_dev_s(&args, &dev, "kcov") != 0) { 574 printf("%s", "Failed to create kcov device"); 575 return; 576 } 577 578 EVENTHANDLER_REGISTER(thread_dtor, kcov_thread_dtor, NULL, 579 EVENTHANDLER_PRI_ANY); 580 } 581 582 SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL);
Cache object: be3efa9fcb678231c7ebc1daa6a0d2cd
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]