FreeBSD/Linux Kernel Cross Reference
sys/dev/kcov.c
1 /* $OpenBSD: kcov.c,v 1.48 2022/01/19 06:46:55 anton Exp $ */
2
3 /*
4 * Copyright (c) 2018 Anton Lindqvist <anton@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/param.h>
20 #include <sys/systm.h>
21 #include <sys/proc.h>
22 #include <sys/kcov.h>
23 #include <sys/malloc.h>
24 #include <sys/mutex.h>
25 #include <sys/pool.h>
26 #include <sys/stdint.h>
27 #include <sys/queue.h>
28
29 /* kcov_vnode() */
30 #include <sys/conf.h>
31 #include <sys/vnode.h>
32 #include <sys/specdev.h>
33
34 #include <uvm/uvm_extern.h>
35
36 #define KCOV_BUF_MEMB_SIZE sizeof(uintptr_t)
37 #define KCOV_BUF_MAX_NMEMB (256 << 10)
38
39 #define KCOV_CMP_CONST 0x1
40 #define KCOV_CMP_SIZE(x) ((x) << 1)
41
42 #define KCOV_STATE_NONE 0
43 #define KCOV_STATE_READY 1
44 #define KCOV_STATE_TRACE 2
45 #define KCOV_STATE_DYING 3
46
47 #define KCOV_STRIDE_TRACE_PC 1
48 #define KCOV_STRIDE_TRACE_CMP 4
49
50 /*
51 * Coverage structure.
52 *
53 * Locking:
54 * I immutable after creation
55 * M kcov_mtx
56 * a atomic operations
57 */
58 struct kcov_dev {
59 int kd_state; /* [M] */
60 int kd_mode; /* [M] */
61 int kd_unit; /* [I] D_CLONE unique device minor */
62 int kd_intr; /* [M] currently used in interrupt */
63 uintptr_t *kd_buf; /* [a] traced coverage */
64 size_t kd_nmemb; /* [I] */
65 size_t kd_size; /* [I] */
66
67 struct kcov_remote *kd_kr; /* [M] */
68
69 TAILQ_ENTRY(kcov_dev) kd_entry; /* [M] */
70 };
71
72 /*
73 * Remote coverage structure.
74 *
75 * Locking:
76 * I immutable after creation
77 * M kcov_mtx
78 */
79 struct kcov_remote {
80 struct kcov_dev *kr_kd; /* [M] */
81 void *kr_id; /* [I] */
82 int kr_subsystem; /* [I] */
83 int kr_nsections; /* [M] # threads in remote section */
84 int kr_state; /* [M] */
85
86 TAILQ_ENTRY(kcov_remote) kr_entry; /* [M] */
87 };
88
89 /*
90 * Per CPU coverage structure used to track coverage when executing in a remote
91 * interrupt context.
92 *
93 * Locking:
94 * I immutable after creation
95 * M kcov_mtx
96 */
97 struct kcov_cpu {
98 struct kcov_dev kc_kd;
99 struct kcov_dev *kc_kd_save; /* [M] previous kcov_dev */
100 int kc_cpuid; /* [I] cpu number */
101
102 TAILQ_ENTRY(kcov_cpu) kc_entry; /* [I] */
103 };
104
105 void kcovattach(int);
106
107 int kd_init(struct kcov_dev *, unsigned long);
108 void kd_free(struct kcov_dev *);
109 struct kcov_dev *kd_lookup(int);
110 void kd_copy(struct kcov_dev *, struct kcov_dev *);
111
112 struct kcov_remote *kcov_remote_register_locked(int, void *);
113 int kcov_remote_attach(struct kcov_dev *, struct kio_remote_attach *);
114 void kcov_remote_detach(struct kcov_dev *, struct kcov_remote *);
115 void kr_free(struct kcov_remote *);
116 void kr_barrier(struct kcov_remote *);
117 struct kcov_remote *kr_lookup(int, void *);
118
119 static struct kcov_dev *kd_curproc(int);
120 static struct kcov_cpu *kd_curcpu(void);
121 static uint64_t kd_claim(struct kcov_dev *, int, int);
122 static inline int inintr(void);
123
124 TAILQ_HEAD(, kcov_dev) kd_list = TAILQ_HEAD_INITIALIZER(kd_list);
125 TAILQ_HEAD(, kcov_remote) kr_list = TAILQ_HEAD_INITIALIZER(kr_list);
126 TAILQ_HEAD(, kcov_cpu) kc_list = TAILQ_HEAD_INITIALIZER(kc_list);
127
128 int kcov_cold = 1;
129 int kr_cold = 1;
130 struct mutex kcov_mtx = MUTEX_INITIALIZER(IPL_MPFLOOR);
131 struct pool kr_pool;
132
133 /*
134 * Compiling the kernel with the `-fsanitize-coverage=trace-pc' option will
135 * cause the following function to be called upon function entry and before
136 * each block instructions that maps to a single line in the original source
137 * code.
138 *
139 * If kcov is enabled for the current thread, the kernel program counter will
140 * be stored in its corresponding coverage buffer.
141 */
142 void
143 __sanitizer_cov_trace_pc(void)
144 {
145 struct kcov_dev *kd;
146 uint64_t idx;
147
148 kd = kd_curproc(KCOV_MODE_TRACE_PC);
149 if (kd == NULL)
150 return;
151
152 if ((idx = kd_claim(kd, KCOV_STRIDE_TRACE_PC, 1)))
153 kd->kd_buf[idx] = (uintptr_t)__builtin_return_address(0);
154 }
155
156 /*
157 * Compiling the kernel with the `-fsanitize-coverage=trace-cmp' option will
158 * cause the following function to be called upon integer comparisons and switch
159 * statements.
160 *
161 * If kcov is enabled for the current thread, the comparison will be stored in
162 * its corresponding coverage buffer.
163 */
164 void
165 trace_cmp(struct kcov_dev *kd, uint64_t type, uint64_t arg1, uint64_t arg2,
166 uintptr_t pc)
167 {
168 uint64_t idx;
169
170 if ((idx = kd_claim(kd, KCOV_STRIDE_TRACE_CMP, 1))) {
171 kd->kd_buf[idx] = type;
172 kd->kd_buf[idx + 1] = arg1;
173 kd->kd_buf[idx + 2] = arg2;
174 kd->kd_buf[idx + 3] = pc;
175 }
176 }
177
178 #define TRACE_CMP(type, arg1, arg2) do { \
179 struct kcov_dev *kd; \
180 if ((kd = kd_curproc(KCOV_MODE_TRACE_CMP)) == NULL) \
181 return; \
182 trace_cmp(kd, (type), (arg1), (arg2), \
183 (uintptr_t)__builtin_return_address(0)); \
184 } while (0)
185
186 void
187 __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2)
188 {
189 TRACE_CMP(KCOV_CMP_SIZE(0), arg1, arg2);
190 }
191
192 void
193 __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2)
194 {
195 TRACE_CMP(KCOV_CMP_SIZE(1), arg1, arg2);
196 }
197
198 void
199 __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2)
200 {
201 TRACE_CMP(KCOV_CMP_SIZE(2), arg1, arg2);
202 }
203
204 void
205 __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2)
206 {
207 TRACE_CMP(KCOV_CMP_SIZE(3), arg1, arg2);
208 }
209
210 void
211 __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2)
212 {
213 TRACE_CMP(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2);
214 }
215
216 void
217 __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2)
218 {
219 TRACE_CMP(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2);
220 }
221
222 void
223 __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2)
224 {
225 TRACE_CMP(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2);
226 }
227
228 void
229 __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2)
230 {
231 TRACE_CMP(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2);
232 }
233
234 void
235 __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases)
236 {
237 struct kcov_dev *kd;
238 uint64_t i, nbits, ncases, type;
239 uintptr_t pc;
240
241 kd = kd_curproc(KCOV_MODE_TRACE_CMP);
242 if (kd == NULL)
243 return;
244
245 pc = (uintptr_t)__builtin_return_address(0);
246 ncases = cases[0];
247 nbits = cases[1];
248
249 switch (nbits) {
250 case 8:
251 type = KCOV_CMP_SIZE(0);
252 break;
253 case 16:
254 type = KCOV_CMP_SIZE(1);
255 break;
256 case 32:
257 type = KCOV_CMP_SIZE(2);
258 break;
259 case 64:
260 type = KCOV_CMP_SIZE(3);
261 break;
262 default:
263 return;
264 }
265 type |= KCOV_CMP_CONST;
266
267 for (i = 0; i < ncases; i++)
268 trace_cmp(kd, type, cases[i + 2], val, pc);
269 }
270
271 void
272 kcovattach(int count)
273 {
274 struct kcov_cpu *kc;
275 int error, i;
276
277 pool_init(&kr_pool, sizeof(struct kcov_remote), 0, IPL_MPFLOOR, PR_WAITOK,
278 "kcovpl", NULL);
279
280 kc = mallocarray(ncpusfound, sizeof(*kc), M_DEVBUF, M_WAITOK | M_ZERO);
281 mtx_enter(&kcov_mtx);
282 for (i = 0; i < ncpusfound; i++) {
283 kc[i].kc_cpuid = i;
284 error = kd_init(&kc[i].kc_kd, KCOV_BUF_MAX_NMEMB);
285 KASSERT(error == 0);
286 TAILQ_INSERT_TAIL(&kc_list, &kc[i], kc_entry);
287 }
288 mtx_leave(&kcov_mtx);
289
290 kr_cold = 0;
291 }
292
293 int
294 kcovopen(dev_t dev, int flag, int mode, struct proc *p)
295 {
296 struct kcov_dev *kd;
297
298 kd = malloc(sizeof(*kd), M_SUBPROC, M_WAITOK | M_ZERO);
299 kd->kd_unit = minor(dev);
300 mtx_enter(&kcov_mtx);
301 KASSERT(kd_lookup(kd->kd_unit) == NULL);
302 TAILQ_INSERT_TAIL(&kd_list, kd, kd_entry);
303 if (kcov_cold)
304 kcov_cold = 0;
305 mtx_leave(&kcov_mtx);
306 return (0);
307 }
308
309 int
310 kcovclose(dev_t dev, int flag, int mode, struct proc *p)
311 {
312 struct kcov_dev *kd;
313
314 mtx_enter(&kcov_mtx);
315
316 kd = kd_lookup(minor(dev));
317 if (kd == NULL) {
318 mtx_leave(&kcov_mtx);
319 return (ENXIO);
320 }
321
322 TAILQ_REMOVE(&kd_list, kd, kd_entry);
323 if (kd->kd_state == KCOV_STATE_TRACE && kd->kd_kr == NULL) {
324 /*
325 * Another thread is currently using the kcov descriptor,
326 * postpone freeing to kcov_exit().
327 */
328 kd->kd_state = KCOV_STATE_DYING;
329 kd->kd_mode = KCOV_MODE_NONE;
330 } else {
331 kd_free(kd);
332 }
333
334 mtx_leave(&kcov_mtx);
335 return (0);
336 }
337
338 int
339 kcovioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
340 {
341 struct kcov_dev *kd;
342 int mode;
343 int error = 0;
344
345 mtx_enter(&kcov_mtx);
346
347 kd = kd_lookup(minor(dev));
348 if (kd == NULL) {
349 mtx_leave(&kcov_mtx);
350 return (ENXIO);
351 }
352
353 switch (cmd) {
354 case KIOSETBUFSIZE:
355 error = kd_init(kd, *((unsigned long *)data));
356 break;
357 case KIOENABLE:
358 /* Only one kcov descriptor can be enabled per thread. */
359 if (p->p_kd != NULL) {
360 error = EBUSY;
361 break;
362 }
363 if (kd->kd_state != KCOV_STATE_READY) {
364 error = ENXIO;
365 break;
366 }
367 mode = *((int *)data);
368 if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) {
369 error = EINVAL;
370 break;
371 }
372 kd->kd_state = KCOV_STATE_TRACE;
373 kd->kd_mode = mode;
374 /* Remote coverage is mutually exclusive. */
375 if (kd->kd_kr == NULL)
376 p->p_kd = kd;
377 break;
378 case KIODISABLE:
379 /* Only the enabled thread may disable itself. */
380 if ((p->p_kd != kd && kd->kd_kr == NULL)) {
381 error = EPERM;
382 break;
383 }
384 if (kd->kd_state != KCOV_STATE_TRACE) {
385 error = ENXIO;
386 break;
387 }
388 kd->kd_state = KCOV_STATE_READY;
389 kd->kd_mode = KCOV_MODE_NONE;
390 if (kd->kd_kr != NULL)
391 kr_barrier(kd->kd_kr);
392 p->p_kd = NULL;
393 break;
394 case KIOREMOTEATTACH:
395 error = kcov_remote_attach(kd,
396 (struct kio_remote_attach *)data);
397 break;
398 default:
399 error = ENOTTY;
400 }
401 mtx_leave(&kcov_mtx);
402
403 return (error);
404 }
405
406 paddr_t
407 kcovmmap(dev_t dev, off_t offset, int prot)
408 {
409 struct kcov_dev *kd;
410 paddr_t pa = -1;
411 vaddr_t va;
412
413 mtx_enter(&kcov_mtx);
414
415 kd = kd_lookup(minor(dev));
416 if (kd == NULL)
417 goto out;
418
419 if (offset < 0 || offset >= kd->kd_nmemb * KCOV_BUF_MEMB_SIZE)
420 goto out;
421
422 va = (vaddr_t)kd->kd_buf + offset;
423 if (pmap_extract(pmap_kernel(), va, &pa) == FALSE)
424 pa = -1;
425
426 out:
427 mtx_leave(&kcov_mtx);
428 return (pa);
429 }
430
431 void
432 kcov_exit(struct proc *p)
433 {
434 struct kcov_dev *kd;
435
436 mtx_enter(&kcov_mtx);
437
438 kd = p->p_kd;
439 if (kd == NULL) {
440 mtx_leave(&kcov_mtx);
441 return;
442 }
443
444 if (kd->kd_state == KCOV_STATE_DYING) {
445 p->p_kd = NULL;
446 kd_free(kd);
447 } else {
448 kd->kd_state = KCOV_STATE_READY;
449 kd->kd_mode = KCOV_MODE_NONE;
450 if (kd->kd_kr != NULL)
451 kr_barrier(kd->kd_kr);
452 p->p_kd = NULL;
453 }
454
455 mtx_leave(&kcov_mtx);
456 }
457
458 /*
459 * Returns non-zero if the given vnode refers to a kcov device.
460 */
461 int
462 kcov_vnode(struct vnode *vp)
463 {
464 return (vp->v_type == VCHR &&
465 cdevsw[major(vp->v_rdev)].d_open == kcovopen);
466 }
467
468 struct kcov_dev *
469 kd_lookup(int unit)
470 {
471 struct kcov_dev *kd;
472
473 MUTEX_ASSERT_LOCKED(&kcov_mtx);
474
475 TAILQ_FOREACH(kd, &kd_list, kd_entry) {
476 if (kd->kd_unit == unit)
477 return (kd);
478 }
479 return (NULL);
480 }
481
482 void
483 kd_copy(struct kcov_dev *dst, struct kcov_dev *src)
484 {
485 uint64_t idx, nmemb;
486 int stride;
487
488 MUTEX_ASSERT_LOCKED(&kcov_mtx);
489 KASSERT(dst->kd_mode == src->kd_mode);
490
491 nmemb = src->kd_buf[0];
492 if (nmemb == 0)
493 return;
494 stride = src->kd_mode == KCOV_MODE_TRACE_CMP ? KCOV_STRIDE_TRACE_CMP :
495 KCOV_STRIDE_TRACE_PC;
496 idx = kd_claim(dst, stride, nmemb);
497 if (idx == 0)
498 return;
499 memcpy(&dst->kd_buf[idx], &src->kd_buf[1],
500 stride * nmemb * KCOV_BUF_MEMB_SIZE);
501 }
502
503 int
504 kd_init(struct kcov_dev *kd, unsigned long nmemb)
505 {
506 void *buf;
507 size_t size;
508 int error;
509
510 KASSERT(kd->kd_buf == NULL);
511
512 if (kd->kd_state != KCOV_STATE_NONE)
513 return (EBUSY);
514
515 if (nmemb == 0 || nmemb > KCOV_BUF_MAX_NMEMB)
516 return (EINVAL);
517
518 size = roundup(nmemb * KCOV_BUF_MEMB_SIZE, PAGE_SIZE);
519 mtx_leave(&kcov_mtx);
520 buf = km_alloc(size, &kv_any, &kp_zero, &kd_waitok);
521 if (buf == NULL) {
522 error = ENOMEM;
523 goto err;
524 }
525 /* km_malloc() can sleep, ensure the race was won. */
526 if (kd->kd_state != KCOV_STATE_NONE) {
527 error = EBUSY;
528 goto err;
529 }
530 mtx_enter(&kcov_mtx);
531 kd->kd_buf = buf;
532 /* The first element is reserved to hold the number of used elements. */
533 kd->kd_nmemb = nmemb - 1;
534 kd->kd_size = size;
535 kd->kd_state = KCOV_STATE_READY;
536 return (0);
537
538 err:
539 if (buf != NULL)
540 km_free(buf, size, &kv_any, &kp_zero);
541 mtx_enter(&kcov_mtx);
542 return (error);
543 }
544
545 void
546 kd_free(struct kcov_dev *kd)
547 {
548 struct kcov_remote *kr;
549
550 MUTEX_ASSERT_LOCKED(&kcov_mtx);
551
552 kr = kd->kd_kr;
553 if (kr != NULL)
554 kcov_remote_detach(kd, kr);
555
556 if (kd->kd_buf != NULL) {
557 mtx_leave(&kcov_mtx);
558 km_free(kd->kd_buf, kd->kd_size, &kv_any, &kp_zero);
559 mtx_enter(&kcov_mtx);
560 }
561 free(kd, M_SUBPROC, sizeof(*kd));
562 }
563
564 static struct kcov_dev *
565 kd_curproc(int mode)
566 {
567 struct kcov_dev *kd;
568
569 /*
570 * Do not trace before kcovopen() has been called at least once.
571 * At this point, all secondary CPUs have booted and accessing curcpu()
572 * is safe.
573 */
574 if (__predict_false(kcov_cold))
575 return (NULL);
576
577 kd = curproc->p_kd;
578 if (__predict_true(kd == NULL) || kd->kd_mode != mode)
579 return (NULL);
580
581 /*
582 * Do not trace if the kernel has panicked. This could happen if curproc
583 * had kcov enabled while panicking.
584 */
585 if (__predict_false(panicstr || db_active))
586 return (NULL);
587
588 /* Do not trace in interrupt context unless this is a remote section. */
589 if (inintr() && kd->kd_intr == 0)
590 return (NULL);
591
592 return (kd);
593
594 }
595
596 static struct kcov_cpu *
597 kd_curcpu(void)
598 {
599 struct kcov_cpu *kc;
600 unsigned int cpuid = cpu_number();
601
602 TAILQ_FOREACH(kc, &kc_list, kc_entry) {
603 if (kc->kc_cpuid == cpuid)
604 return (kc);
605 }
606 return (NULL);
607 }
608
609 /*
610 * Claim stride times nmemb number of elements in the coverage buffer. Returns
611 * the index of the first claimed element. If the claim cannot be fulfilled,
612 * zero is returned.
613 */
614 static uint64_t
615 kd_claim(struct kcov_dev *kd, int stride, int nmemb)
616 {
617 uint64_t idx, was;
618
619 idx = kd->kd_buf[0];
620 for (;;) {
621 if (stride * (idx + nmemb) > kd->kd_nmemb)
622 return (0);
623
624 was = atomic_cas_ulong(&kd->kd_buf[0], idx, idx + nmemb);
625 if (was == idx)
626 return (idx * stride + 1);
627 idx = was;
628 }
629 }
630
631 static inline int
632 inintr(void)
633 {
634 #if defined(__amd64__) || defined(__arm__) || defined(__arm64__) || \
635 defined(__i386__)
636 return (curcpu()->ci_idepth > 0);
637 #else
638 return (0);
639 #endif
640 }
641
642 void
643 kcov_remote_enter(int subsystem, void *id)
644 {
645 struct kcov_cpu *kc;
646 struct kcov_dev *kd;
647 struct kcov_remote *kr;
648 struct proc *p;
649
650 mtx_enter(&kcov_mtx);
651 kr = kr_lookup(subsystem, id);
652 if (kr == NULL || kr->kr_state != KCOV_STATE_READY)
653 goto out;
654 kd = kr->kr_kd;
655 if (kd == NULL || kd->kd_state != KCOV_STATE_TRACE)
656 goto out;
657 p = curproc;
658 if (inintr()) {
659 /*
660 * XXX we only expect to be called from softclock interrupts at
661 * this point.
662 */
663 kc = kd_curcpu();
664 if (kc == NULL || kc->kc_kd.kd_intr == 1)
665 goto out;
666 kc->kc_kd.kd_state = KCOV_STATE_TRACE;
667 kc->kc_kd.kd_mode = kd->kd_mode;
668 kc->kc_kd.kd_intr = 1;
669 kc->kc_kd_save = p->p_kd;
670 kd = &kc->kc_kd;
671 /* Reset coverage buffer. */
672 kd->kd_buf[0] = 0;
673 } else {
674 KASSERT(p->p_kd == NULL);
675 }
676 kr->kr_nsections++;
677 p->p_kd = kd;
678
679 out:
680 mtx_leave(&kcov_mtx);
681 }
682
683 void
684 kcov_remote_leave(int subsystem, void *id)
685 {
686 struct kcov_cpu *kc;
687 struct kcov_remote *kr;
688 struct proc *p;
689
690 mtx_enter(&kcov_mtx);
691 p = curproc;
692 if (p->p_kd == NULL)
693 goto out;
694 kr = kr_lookup(subsystem, id);
695 if (kr == NULL)
696 goto out;
697 if (inintr()) {
698 kc = kd_curcpu();
699 if (kc == NULL || kc->kc_kd.kd_intr == 0)
700 goto out;
701
702 /*
703 * Stop writing to the coverage buffer associated with this CPU
704 * before copying its contents.
705 */
706 p->p_kd = kc->kc_kd_save;
707 kc->kc_kd_save = NULL;
708
709 kd_copy(kr->kr_kd, &kc->kc_kd);
710 kc->kc_kd.kd_state = KCOV_STATE_READY;
711 kc->kc_kd.kd_mode = KCOV_MODE_NONE;
712 kc->kc_kd.kd_intr = 0;
713 } else {
714 KASSERT(p->p_kd == kr->kr_kd);
715 p->p_kd = NULL;
716 }
717 if (--kr->kr_nsections == 0)
718 wakeup(kr);
719 out:
720 mtx_leave(&kcov_mtx);
721 }
722
723 void
724 kcov_remote_register(int subsystem, void *id)
725 {
726 mtx_enter(&kcov_mtx);
727 kcov_remote_register_locked(subsystem, id);
728 mtx_leave(&kcov_mtx);
729 }
730
731 void
732 kcov_remote_unregister(int subsystem, void *id)
733 {
734 struct kcov_remote *kr;
735
736 mtx_enter(&kcov_mtx);
737 kr = kr_lookup(subsystem, id);
738 if (kr != NULL)
739 kr_free(kr);
740 mtx_leave(&kcov_mtx);
741 }
742
743 struct kcov_remote *
744 kcov_remote_register_locked(int subsystem, void *id)
745 {
746 struct kcov_remote *kr, *tmp;
747
748 /* Do not allow registrations before the pool is initialized. */
749 KASSERT(kr_cold == 0);
750
751 /*
752 * Temporarily release the mutex since the allocation could end up
753 * sleeping.
754 */
755 mtx_leave(&kcov_mtx);
756 kr = pool_get(&kr_pool, PR_WAITOK | PR_ZERO);
757 kr->kr_subsystem = subsystem;
758 kr->kr_id = id;
759 kr->kr_state = KCOV_STATE_NONE;
760 mtx_enter(&kcov_mtx);
761
762 for (;;) {
763 tmp = kr_lookup(subsystem, id);
764 if (tmp == NULL)
765 break;
766 if (tmp->kr_state != KCOV_STATE_DYING) {
767 pool_put(&kr_pool, kr);
768 return (NULL);
769 }
770 /*
771 * The remote could already be deregistered while another
772 * thread is currently inside a kcov remote section.
773 */
774 msleep_nsec(tmp, &kcov_mtx, PWAIT, "kcov", INFSLP);
775 }
776 TAILQ_INSERT_TAIL(&kr_list, kr, kr_entry);
777 return (kr);
778 }
779
780 int
781 kcov_remote_attach(struct kcov_dev *kd, struct kio_remote_attach *arg)
782 {
783 struct kcov_remote *kr = NULL;
784
785 MUTEX_ASSERT_LOCKED(&kcov_mtx);
786
787 if (kd->kd_state != KCOV_STATE_READY)
788 return (ENXIO);
789
790 if (arg->subsystem == KCOV_REMOTE_COMMON) {
791 kr = kcov_remote_register_locked(KCOV_REMOTE_COMMON,
792 curproc->p_p);
793 if (kr == NULL)
794 return (EBUSY);
795 } else {
796 return (EINVAL);
797 }
798
799 kr->kr_state = KCOV_STATE_READY;
800 kr->kr_kd = kd;
801 kd->kd_kr = kr;
802 return (0);
803 }
804
805 void
806 kcov_remote_detach(struct kcov_dev *kd, struct kcov_remote *kr)
807 {
808 MUTEX_ASSERT_LOCKED(&kcov_mtx);
809
810 KASSERT(kd == kr->kr_kd);
811 if (kr->kr_subsystem == KCOV_REMOTE_COMMON) {
812 kr_free(kr);
813 } else {
814 kr->kr_state = KCOV_STATE_NONE;
815 kr_barrier(kr);
816 kd->kd_kr = NULL;
817 kr->kr_kd = NULL;
818 }
819 }
820
821 void
822 kr_free(struct kcov_remote *kr)
823 {
824 MUTEX_ASSERT_LOCKED(&kcov_mtx);
825
826 kr->kr_state = KCOV_STATE_DYING;
827 kr_barrier(kr);
828 if (kr->kr_kd != NULL)
829 kr->kr_kd->kd_kr = NULL;
830 kr->kr_kd = NULL;
831 TAILQ_REMOVE(&kr_list, kr, kr_entry);
832 /* Notify thread(s) waiting in kcov_remote_register(). */
833 wakeup(kr);
834 pool_put(&kr_pool, kr);
835 }
836
837 void
838 kr_barrier(struct kcov_remote *kr)
839 {
840 MUTEX_ASSERT_LOCKED(&kcov_mtx);
841
842 while (kr->kr_nsections > 0)
843 msleep_nsec(kr, &kcov_mtx, PWAIT, "kcovbar", INFSLP);
844 }
845
846 struct kcov_remote *
847 kr_lookup(int subsystem, void *id)
848 {
849 struct kcov_remote *kr;
850
851 MUTEX_ASSERT_LOCKED(&kcov_mtx);
852
853 TAILQ_FOREACH(kr, &kr_list, kr_entry) {
854 if (kr->kr_subsystem == subsystem && kr->kr_id == id)
855 return (kr);
856 }
857 return (NULL);
858 }
Cache object: d4783cf1eabaa0a153f1d0058477bb85
|