kcov.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $OpenBSD: kcov.c,v 1.48 2022/01/19 06:46:55 anton Exp $ */
    2 
    3 /*
    4  * Copyright (c) 2018 Anton Lindqvist <anton@openbsd.org>
    5  *
    6  * Permission to use, copy, modify, and distribute this software for any
    7  * purpose with or without fee is hereby granted, provided that the above
    8  * copyright notice and this permission notice appear in all copies.
    9  *
   10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
   12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
   13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
   14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
   17  */
   18 
   19 #include <sys/param.h>
   20 #include <sys/systm.h>
   21 #include <sys/proc.h>
   22 #include <sys/kcov.h>
   23 #include <sys/malloc.h>
   24 #include <sys/mutex.h>
   25 #include <sys/pool.h>
   26 #include <sys/stdint.h>
   27 #include <sys/queue.h>
   28 
   29 /* kcov_vnode() */
   30 #include <sys/conf.h>
   31 #include <sys/vnode.h>
   32 #include <sys/specdev.h>
   33 
   34 #include <uvm/uvm_extern.h>
   35 
   36 #define KCOV_BUF_MEMB_SIZE      sizeof(uintptr_t)
   37 #define KCOV_BUF_MAX_NMEMB      (256 << 10)
   38 
   39 #define KCOV_CMP_CONST          0x1
   40 #define KCOV_CMP_SIZE(x)        ((x) << 1)
   41 
   42 #define KCOV_STATE_NONE         0
   43 #define KCOV_STATE_READY        1
   44 #define KCOV_STATE_TRACE        2
   45 #define KCOV_STATE_DYING        3
   46 
   47 #define KCOV_STRIDE_TRACE_PC    1
   48 #define KCOV_STRIDE_TRACE_CMP   4
   49 
   50 /*
   51  * Coverage structure.
   52  *
   53  * Locking:
   54  *      I       immutable after creation
   55  *      M       kcov_mtx
   56  *      a       atomic operations
   57  */
   58 struct kcov_dev {
   59         int              kd_state;      /* [M] */
   60         int              kd_mode;       /* [M] */
   61         int              kd_unit;       /* [I] D_CLONE unique device minor */
   62         int              kd_intr;       /* [M] currently used in interrupt */
   63         uintptr_t       *kd_buf;        /* [a] traced coverage */
   64         size_t           kd_nmemb;      /* [I] */
   65         size_t           kd_size;       /* [I] */
   66 
   67         struct kcov_remote *kd_kr;      /* [M] */
   68 
   69         TAILQ_ENTRY(kcov_dev)   kd_entry;       /* [M] */
   70 };
   71 
   72 /*
   73  * Remote coverage structure.
   74  *
   75  * Locking:
   76  *      I       immutable after creation
   77  *      M       kcov_mtx
   78  */
   79 struct kcov_remote {
   80         struct kcov_dev *kr_kd; /* [M] */
   81         void *kr_id;            /* [I] */
   82         int kr_subsystem;       /* [I] */
   83         int kr_nsections;       /* [M] # threads in remote section */
   84         int kr_state;           /* [M] */
   85 
   86         TAILQ_ENTRY(kcov_remote) kr_entry;      /* [M] */
   87 };
   88 
   89 /*
   90  * Per CPU coverage structure used to track coverage when executing in a remote
   91  * interrupt context.
   92  *
   93  * Locking:
   94  *      I       immutable after creation
   95  *      M       kcov_mtx
   96  */
   97 struct kcov_cpu {
   98         struct kcov_dev  kc_kd;
   99         struct kcov_dev *kc_kd_save;    /* [M] previous kcov_dev */
  100         int kc_cpuid;                   /* [I] cpu number */
  101 
  102         TAILQ_ENTRY(kcov_cpu) kc_entry; /* [I] */
  103 };
  104 
  105 void kcovattach(int);
  106 
  107 int kd_init(struct kcov_dev *, unsigned long);
  108 void kd_free(struct kcov_dev *);
  109 struct kcov_dev *kd_lookup(int);
  110 void kd_copy(struct kcov_dev *, struct kcov_dev *);
  111 
  112 struct kcov_remote *kcov_remote_register_locked(int, void *);
  113 int kcov_remote_attach(struct kcov_dev *, struct kio_remote_attach *);
  114 void kcov_remote_detach(struct kcov_dev *, struct kcov_remote *);
  115 void kr_free(struct kcov_remote *);
  116 void kr_barrier(struct kcov_remote *);
  117 struct kcov_remote *kr_lookup(int, void *);
  118 
  119 static struct kcov_dev *kd_curproc(int);
  120 static struct kcov_cpu *kd_curcpu(void);
  121 static uint64_t kd_claim(struct kcov_dev *, int, int);
  122 static inline int inintr(void);
  123 
  124 TAILQ_HEAD(, kcov_dev) kd_list = TAILQ_HEAD_INITIALIZER(kd_list);
  125 TAILQ_HEAD(, kcov_remote) kr_list = TAILQ_HEAD_INITIALIZER(kr_list);
  126 TAILQ_HEAD(, kcov_cpu) kc_list = TAILQ_HEAD_INITIALIZER(kc_list);
  127 
  128 int kcov_cold = 1;
  129 int kr_cold = 1;
  130 struct mutex kcov_mtx = MUTEX_INITIALIZER(IPL_MPFLOOR);
  131 struct pool kr_pool;
  132 
  133 /*
  134  * Compiling the kernel with the `-fsanitize-coverage=trace-pc' option will
  135  * cause the following function to be called upon function entry and before
  136  * each block instructions that maps to a single line in the original source
  137  * code.
  138  *
  139  * If kcov is enabled for the current thread, the kernel program counter will
  140  * be stored in its corresponding coverage buffer.
  141  */
  142 void
  143 __sanitizer_cov_trace_pc(void)
  144 {
  145         struct kcov_dev *kd;
  146         uint64_t idx;
  147 
  148         kd = kd_curproc(KCOV_MODE_TRACE_PC);
  149         if (kd == NULL)
  150                 return;
  151 
  152         if ((idx = kd_claim(kd, KCOV_STRIDE_TRACE_PC, 1)))
  153                 kd->kd_buf[idx] = (uintptr_t)__builtin_return_address(0);
  154 }
  155 
  156 /*
  157  * Compiling the kernel with the `-fsanitize-coverage=trace-cmp' option will
  158  * cause the following function to be called upon integer comparisons and switch
  159  * statements.
  160  *
  161  * If kcov is enabled for the current thread, the comparison will be stored in
  162  * its corresponding coverage buffer.
  163  */
  164 void
  165 trace_cmp(struct kcov_dev *kd, uint64_t type, uint64_t arg1, uint64_t arg2,
  166     uintptr_t pc)
  167 {
  168         uint64_t idx;
  169 
  170         if ((idx = kd_claim(kd, KCOV_STRIDE_TRACE_CMP, 1))) {
  171                 kd->kd_buf[idx] = type;
  172                 kd->kd_buf[idx + 1] = arg1;
  173                 kd->kd_buf[idx + 2] = arg2;
  174                 kd->kd_buf[idx + 3] = pc;
  175         }
  176 }
  177 
  178 #define TRACE_CMP(type, arg1, arg2) do {                                \
  179         struct kcov_dev *kd;                                            \
  180         if ((kd = kd_curproc(KCOV_MODE_TRACE_CMP)) == NULL)             \
  181                 return;                                                 \
  182         trace_cmp(kd, (type), (arg1), (arg2),                           \
  183             (uintptr_t)__builtin_return_address(0));                    \
  184 } while (0)
  185 
  186 void
  187 __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2)
  188 {
  189         TRACE_CMP(KCOV_CMP_SIZE(0), arg1, arg2);
  190 }
  191 
  192 void
  193 __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2)
  194 {
  195         TRACE_CMP(KCOV_CMP_SIZE(1), arg1, arg2);
  196 }
  197 
  198 void
  199 __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2)
  200 {
  201         TRACE_CMP(KCOV_CMP_SIZE(2), arg1, arg2);
  202 }
  203 
  204 void
  205 __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2)
  206 {
  207         TRACE_CMP(KCOV_CMP_SIZE(3), arg1, arg2);
  208 }
  209 
  210 void
  211 __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2)
  212 {
  213         TRACE_CMP(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2);
  214 }
  215 
  216 void
  217 __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2)
  218 {
  219         TRACE_CMP(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2);
  220 }
  221 
  222 void
  223 __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2)
  224 {
  225         TRACE_CMP(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2);
  226 }
  227 
  228 void
  229 __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2)
  230 {
  231         TRACE_CMP(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2);
  232 }
  233 
  234 void
  235 __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases)
  236 {
  237         struct kcov_dev *kd;
  238         uint64_t i, nbits, ncases, type;
  239         uintptr_t pc;
  240 
  241         kd = kd_curproc(KCOV_MODE_TRACE_CMP);
  242         if (kd == NULL)
  243                 return;
  244 
  245         pc = (uintptr_t)__builtin_return_address(0);
  246         ncases = cases[0];
  247         nbits = cases[1];
  248 
  249         switch (nbits) {
  250         case 8:
  251                 type = KCOV_CMP_SIZE(0);
  252                 break;
  253         case 16:
  254                 type = KCOV_CMP_SIZE(1);
  255                 break;
  256         case 32:
  257                 type = KCOV_CMP_SIZE(2);
  258                 break;
  259         case 64:
  260                 type = KCOV_CMP_SIZE(3);
  261                 break;
  262         default:
  263                 return;
  264         }
  265         type |= KCOV_CMP_CONST;
  266 
  267         for (i = 0; i < ncases; i++)
  268                 trace_cmp(kd, type, cases[i + 2], val, pc);
  269 }
  270 
  271 void
  272 kcovattach(int count)
  273 {
  274         struct kcov_cpu *kc;
  275         int error, i;
  276 
  277         pool_init(&kr_pool, sizeof(struct kcov_remote), 0, IPL_MPFLOOR, PR_WAITOK,
  278             "kcovpl", NULL);
  279 
  280         kc = mallocarray(ncpusfound, sizeof(*kc), M_DEVBUF, M_WAITOK | M_ZERO);
  281         mtx_enter(&kcov_mtx);
  282         for (i = 0; i < ncpusfound; i++) {
  283                 kc[i].kc_cpuid = i;
  284                 error = kd_init(&kc[i].kc_kd, KCOV_BUF_MAX_NMEMB);
  285                 KASSERT(error == 0);
  286                 TAILQ_INSERT_TAIL(&kc_list, &kc[i], kc_entry);
  287         }
  288         mtx_leave(&kcov_mtx);
  289 
  290         kr_cold = 0;
  291 }
  292 
  293 int
  294 kcovopen(dev_t dev, int flag, int mode, struct proc *p)
  295 {
  296         struct kcov_dev *kd;
  297 
  298         kd = malloc(sizeof(*kd), M_SUBPROC, M_WAITOK | M_ZERO);
  299         kd->kd_unit = minor(dev);
  300         mtx_enter(&kcov_mtx);
  301         KASSERT(kd_lookup(kd->kd_unit) == NULL);
  302         TAILQ_INSERT_TAIL(&kd_list, kd, kd_entry);
  303         if (kcov_cold)
  304                 kcov_cold = 0;
  305         mtx_leave(&kcov_mtx);
  306         return (0);
  307 }
  308 
  309 int
  310 kcovclose(dev_t dev, int flag, int mode, struct proc *p)
  311 {
  312         struct kcov_dev *kd;
  313 
  314         mtx_enter(&kcov_mtx);
  315 
  316         kd = kd_lookup(minor(dev));
  317         if (kd == NULL) {
  318                 mtx_leave(&kcov_mtx);
  319                 return (ENXIO);
  320         }
  321 
  322         TAILQ_REMOVE(&kd_list, kd, kd_entry);
  323         if (kd->kd_state == KCOV_STATE_TRACE && kd->kd_kr == NULL) {
  324                 /*
  325                  * Another thread is currently using the kcov descriptor,
  326                  * postpone freeing to kcov_exit().
  327                  */
  328                 kd->kd_state = KCOV_STATE_DYING;
  329                 kd->kd_mode = KCOV_MODE_NONE;
  330         } else {
  331                 kd_free(kd);
  332         }
  333 
  334         mtx_leave(&kcov_mtx);
  335         return (0);
  336 }
  337 
  338 int
  339 kcovioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
  340 {
  341         struct kcov_dev *kd;
  342         int mode;
  343         int error = 0;
  344 
  345         mtx_enter(&kcov_mtx);
  346 
  347         kd = kd_lookup(minor(dev));
  348         if (kd == NULL) {
  349                 mtx_leave(&kcov_mtx);
  350                 return (ENXIO);
  351         }
  352 
  353         switch (cmd) {
  354         case KIOSETBUFSIZE:
  355                 error = kd_init(kd, *((unsigned long *)data));
  356                 break;
  357         case KIOENABLE:
  358                 /* Only one kcov descriptor can be enabled per thread. */
  359                 if (p->p_kd != NULL) {
  360                         error = EBUSY;
  361                         break;
  362                 }
  363                 if (kd->kd_state != KCOV_STATE_READY) {
  364                         error = ENXIO;
  365                         break;
  366                 }
  367                 mode = *((int *)data);
  368                 if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) {
  369                         error = EINVAL;
  370                         break;
  371                 }
  372                 kd->kd_state = KCOV_STATE_TRACE;
  373                 kd->kd_mode = mode;
  374                 /* Remote coverage is mutually exclusive. */
  375                 if (kd->kd_kr == NULL)
  376                         p->p_kd = kd;
  377                 break;
  378         case KIODISABLE:
  379                 /* Only the enabled thread may disable itself. */
  380                 if ((p->p_kd != kd && kd->kd_kr == NULL)) {
  381                         error = EPERM;
  382                         break;
  383                 }
  384                 if (kd->kd_state != KCOV_STATE_TRACE) {
  385                         error = ENXIO;
  386                         break;
  387                 }
  388                 kd->kd_state = KCOV_STATE_READY;
  389                 kd->kd_mode = KCOV_MODE_NONE;
  390                 if (kd->kd_kr != NULL)
  391                         kr_barrier(kd->kd_kr);
  392                 p->p_kd = NULL;
  393                 break;
  394         case KIOREMOTEATTACH:
  395                 error = kcov_remote_attach(kd,
  396                     (struct kio_remote_attach *)data);
  397                 break;
  398         default:
  399                 error = ENOTTY;
  400         }
  401         mtx_leave(&kcov_mtx);
  402 
  403         return (error);
  404 }
  405 
  406 paddr_t
  407 kcovmmap(dev_t dev, off_t offset, int prot)
  408 {
  409         struct kcov_dev *kd;
  410         paddr_t pa = -1;
  411         vaddr_t va;
  412 
  413         mtx_enter(&kcov_mtx);
  414 
  415         kd = kd_lookup(minor(dev));
  416         if (kd == NULL)
  417                 goto out;
  418 
  419         if (offset < 0 || offset >= kd->kd_nmemb * KCOV_BUF_MEMB_SIZE)
  420                 goto out;
  421 
  422         va = (vaddr_t)kd->kd_buf + offset;
  423         if (pmap_extract(pmap_kernel(), va, &pa) == FALSE)
  424                 pa = -1;
  425 
  426 out:
  427         mtx_leave(&kcov_mtx);
  428         return (pa);
  429 }
  430 
  431 void
  432 kcov_exit(struct proc *p)
  433 {
  434         struct kcov_dev *kd;
  435 
  436         mtx_enter(&kcov_mtx);
  437 
  438         kd = p->p_kd;
  439         if (kd == NULL) {
  440                 mtx_leave(&kcov_mtx);
  441                 return;
  442         }
  443 
  444         if (kd->kd_state == KCOV_STATE_DYING) {
  445                 p->p_kd = NULL;
  446                 kd_free(kd);
  447         } else {
  448                 kd->kd_state = KCOV_STATE_READY;
  449                 kd->kd_mode = KCOV_MODE_NONE;
  450                 if (kd->kd_kr != NULL)
  451                         kr_barrier(kd->kd_kr);
  452                 p->p_kd = NULL;
  453         }
  454 
  455         mtx_leave(&kcov_mtx);
  456 }
  457 
  458 /*
  459  * Returns non-zero if the given vnode refers to a kcov device.
  460  */
  461 int
  462 kcov_vnode(struct vnode *vp)
  463 {
  464         return (vp->v_type == VCHR &&
  465             cdevsw[major(vp->v_rdev)].d_open == kcovopen);
  466 }
  467 
  468 struct kcov_dev *
  469 kd_lookup(int unit)
  470 {
  471         struct kcov_dev *kd;
  472 
  473         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  474 
  475         TAILQ_FOREACH(kd, &kd_list, kd_entry) {
  476                 if (kd->kd_unit == unit)
  477                         return (kd);
  478         }
  479         return (NULL);
  480 }
  481 
  482 void
  483 kd_copy(struct kcov_dev *dst, struct kcov_dev *src)
  484 {
  485         uint64_t idx, nmemb;
  486         int stride;
  487 
  488         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  489         KASSERT(dst->kd_mode == src->kd_mode);
  490 
  491         nmemb = src->kd_buf[0];
  492         if (nmemb == 0)
  493                 return;
  494         stride = src->kd_mode == KCOV_MODE_TRACE_CMP ? KCOV_STRIDE_TRACE_CMP :
  495             KCOV_STRIDE_TRACE_PC;
  496         idx = kd_claim(dst, stride, nmemb);
  497         if (idx == 0)
  498                 return;
  499         memcpy(&dst->kd_buf[idx], &src->kd_buf[1],
  500             stride * nmemb * KCOV_BUF_MEMB_SIZE);
  501 }
  502 
  503 int
  504 kd_init(struct kcov_dev *kd, unsigned long nmemb)
  505 {
  506         void *buf;
  507         size_t size;
  508         int error;
  509 
  510         KASSERT(kd->kd_buf == NULL);
  511 
  512         if (kd->kd_state != KCOV_STATE_NONE)
  513                 return (EBUSY);
  514 
  515         if (nmemb == 0 || nmemb > KCOV_BUF_MAX_NMEMB)
  516                 return (EINVAL);
  517 
  518         size = roundup(nmemb * KCOV_BUF_MEMB_SIZE, PAGE_SIZE);
  519         mtx_leave(&kcov_mtx);
  520         buf = km_alloc(size, &kv_any, &kp_zero, &kd_waitok);
  521         if (buf == NULL) {
  522                 error = ENOMEM;
  523                 goto err;
  524         }
  525         /* km_malloc() can sleep, ensure the race was won. */
  526         if (kd->kd_state != KCOV_STATE_NONE) {
  527                 error = EBUSY;
  528                 goto err;
  529         }
  530         mtx_enter(&kcov_mtx);
  531         kd->kd_buf = buf;
  532         /* The first element is reserved to hold the number of used elements. */
  533         kd->kd_nmemb = nmemb - 1;
  534         kd->kd_size = size;
  535         kd->kd_state = KCOV_STATE_READY;
  536         return (0);
  537 
  538 err:
  539         if (buf != NULL)
  540                 km_free(buf, size, &kv_any, &kp_zero);
  541         mtx_enter(&kcov_mtx);
  542         return (error);
  543 }
  544 
  545 void
  546 kd_free(struct kcov_dev *kd)
  547 {
  548         struct kcov_remote *kr;
  549 
  550         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  551 
  552         kr = kd->kd_kr;
  553         if (kr != NULL)
  554                 kcov_remote_detach(kd, kr);
  555 
  556         if (kd->kd_buf != NULL) {
  557                 mtx_leave(&kcov_mtx);
  558                 km_free(kd->kd_buf, kd->kd_size, &kv_any, &kp_zero);
  559                 mtx_enter(&kcov_mtx);
  560         }
  561         free(kd, M_SUBPROC, sizeof(*kd));
  562 }
  563 
  564 static struct kcov_dev *
  565 kd_curproc(int mode)
  566 {
  567         struct kcov_dev *kd;
  568 
  569         /*
  570          * Do not trace before kcovopen() has been called at least once.
  571          * At this point, all secondary CPUs have booted and accessing curcpu()
  572          * is safe.
  573          */
  574         if (__predict_false(kcov_cold))
  575                 return (NULL);
  576 
  577         kd = curproc->p_kd;
  578         if (__predict_true(kd == NULL) || kd->kd_mode != mode)
  579                 return (NULL);
  580 
  581         /*
  582          * Do not trace if the kernel has panicked. This could happen if curproc
  583          * had kcov enabled while panicking.
  584          */
  585         if (__predict_false(panicstr || db_active))
  586                 return (NULL);
  587 
  588         /* Do not trace in interrupt context unless this is a remote section. */
  589         if (inintr() && kd->kd_intr == 0)
  590                 return (NULL);
  591 
  592         return (kd);
  593 
  594 }
  595 
  596 static struct kcov_cpu *
  597 kd_curcpu(void)
  598 {
  599         struct kcov_cpu *kc;
  600         unsigned int cpuid = cpu_number();
  601 
  602         TAILQ_FOREACH(kc, &kc_list, kc_entry) {
  603                 if (kc->kc_cpuid == cpuid)
  604                         return (kc);
  605         }
  606         return (NULL);
  607 }
  608 
  609 /*
  610  * Claim stride times nmemb number of elements in the coverage buffer. Returns
  611  * the index of the first claimed element. If the claim cannot be fulfilled,
  612  * zero is returned.
  613  */
  614 static uint64_t
  615 kd_claim(struct kcov_dev *kd, int stride, int nmemb)
  616 {
  617         uint64_t idx, was;
  618 
  619         idx = kd->kd_buf[0];
  620         for (;;) {
  621                 if (stride * (idx + nmemb) > kd->kd_nmemb)
  622                         return (0);
  623 
  624                 was = atomic_cas_ulong(&kd->kd_buf[0], idx, idx + nmemb);
  625                 if (was == idx)
  626                         return (idx * stride + 1);
  627                 idx = was;
  628         }
  629 }
  630 
  631 static inline int
  632 inintr(void)
  633 {
  634 #if defined(__amd64__) || defined(__arm__) || defined(__arm64__) || \
  635     defined(__i386__)
  636         return (curcpu()->ci_idepth > 0);
  637 #else
  638         return (0);
  639 #endif
  640 }
  641 
  642 void
  643 kcov_remote_enter(int subsystem, void *id)
  644 {
  645         struct kcov_cpu *kc;
  646         struct kcov_dev *kd;
  647         struct kcov_remote *kr;
  648         struct proc *p;
  649 
  650         mtx_enter(&kcov_mtx);
  651         kr = kr_lookup(subsystem, id);
  652         if (kr == NULL || kr->kr_state != KCOV_STATE_READY)
  653                 goto out;
  654         kd = kr->kr_kd;
  655         if (kd == NULL || kd->kd_state != KCOV_STATE_TRACE)
  656                 goto out;
  657         p = curproc;
  658         if (inintr()) {
  659                 /*
  660                  * XXX we only expect to be called from softclock interrupts at
  661                  * this point.
  662                  */
  663                 kc = kd_curcpu();
  664                 if (kc == NULL || kc->kc_kd.kd_intr == 1)
  665                         goto out;
  666                 kc->kc_kd.kd_state = KCOV_STATE_TRACE;
  667                 kc->kc_kd.kd_mode = kd->kd_mode;
  668                 kc->kc_kd.kd_intr = 1;
  669                 kc->kc_kd_save = p->p_kd;
  670                 kd = &kc->kc_kd;
  671                 /* Reset coverage buffer. */
  672                 kd->kd_buf[0] = 0;
  673         } else {
  674                 KASSERT(p->p_kd == NULL);
  675         }
  676         kr->kr_nsections++;
  677         p->p_kd = kd;
  678 
  679 out:
  680         mtx_leave(&kcov_mtx);
  681 }
  682 
  683 void
  684 kcov_remote_leave(int subsystem, void *id)
  685 {
  686         struct kcov_cpu *kc;
  687         struct kcov_remote *kr;
  688         struct proc *p;
  689 
  690         mtx_enter(&kcov_mtx);
  691         p = curproc;
  692         if (p->p_kd == NULL)
  693                 goto out;
  694         kr = kr_lookup(subsystem, id);
  695         if (kr == NULL)
  696                 goto out;
  697         if (inintr()) {
  698                 kc = kd_curcpu();
  699                 if (kc == NULL || kc->kc_kd.kd_intr == 0)
  700                         goto out;
  701 
  702                 /*
  703                  * Stop writing to the coverage buffer associated with this CPU
  704                  * before copying its contents.
  705                  */
  706                 p->p_kd = kc->kc_kd_save;
  707                 kc->kc_kd_save = NULL;
  708 
  709                 kd_copy(kr->kr_kd, &kc->kc_kd);
  710                 kc->kc_kd.kd_state = KCOV_STATE_READY;
  711                 kc->kc_kd.kd_mode = KCOV_MODE_NONE;
  712                 kc->kc_kd.kd_intr = 0;
  713         } else {
  714                 KASSERT(p->p_kd == kr->kr_kd);
  715                 p->p_kd = NULL;
  716         }
  717         if (--kr->kr_nsections == 0)
  718                 wakeup(kr);
  719 out:
  720         mtx_leave(&kcov_mtx);
  721 }
  722 
  723 void
  724 kcov_remote_register(int subsystem, void *id)
  725 {
  726         mtx_enter(&kcov_mtx);
  727         kcov_remote_register_locked(subsystem, id);
  728         mtx_leave(&kcov_mtx);
  729 }
  730 
  731 void
  732 kcov_remote_unregister(int subsystem, void *id)
  733 {
  734         struct kcov_remote *kr;
  735 
  736         mtx_enter(&kcov_mtx);
  737         kr = kr_lookup(subsystem, id);
  738         if (kr != NULL)
  739                 kr_free(kr);
  740         mtx_leave(&kcov_mtx);
  741 }
  742 
  743 struct kcov_remote *
  744 kcov_remote_register_locked(int subsystem, void *id)
  745 {
  746         struct kcov_remote *kr, *tmp;
  747 
  748         /* Do not allow registrations before the pool is initialized. */
  749         KASSERT(kr_cold == 0);
  750 
  751         /*
  752          * Temporarily release the mutex since the allocation could end up
  753          * sleeping.
  754          */
  755         mtx_leave(&kcov_mtx);
  756         kr = pool_get(&kr_pool, PR_WAITOK | PR_ZERO);
  757         kr->kr_subsystem = subsystem;
  758         kr->kr_id = id;
  759         kr->kr_state = KCOV_STATE_NONE;
  760         mtx_enter(&kcov_mtx);
  761 
  762         for (;;) {
  763                 tmp = kr_lookup(subsystem, id);
  764                 if (tmp == NULL)
  765                         break;
  766                 if (tmp->kr_state != KCOV_STATE_DYING) {
  767                         pool_put(&kr_pool, kr);
  768                         return (NULL);
  769                 }
  770                 /*
  771                  * The remote could already be deregistered while another
  772                  * thread is currently inside a kcov remote section.
  773                  */
  774                 msleep_nsec(tmp, &kcov_mtx, PWAIT, "kcov", INFSLP);
  775         }
  776         TAILQ_INSERT_TAIL(&kr_list, kr, kr_entry);
  777         return (kr);
  778 }
  779 
  780 int
  781 kcov_remote_attach(struct kcov_dev *kd, struct kio_remote_attach *arg)
  782 {
  783         struct kcov_remote *kr = NULL;
  784 
  785         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  786 
  787         if (kd->kd_state != KCOV_STATE_READY)
  788                 return (ENXIO);
  789 
  790         if (arg->subsystem == KCOV_REMOTE_COMMON) {
  791                 kr = kcov_remote_register_locked(KCOV_REMOTE_COMMON,
  792                     curproc->p_p);
  793                 if (kr == NULL)
  794                         return (EBUSY);
  795         } else {
  796                 return (EINVAL);
  797         }
  798 
  799         kr->kr_state = KCOV_STATE_READY;
  800         kr->kr_kd = kd;
  801         kd->kd_kr = kr;
  802         return (0);
  803 }
  804 
  805 void
  806 kcov_remote_detach(struct kcov_dev *kd, struct kcov_remote *kr)
  807 {
  808         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  809 
  810         KASSERT(kd == kr->kr_kd);
  811         if (kr->kr_subsystem == KCOV_REMOTE_COMMON) {
  812                 kr_free(kr);
  813         } else {
  814                 kr->kr_state = KCOV_STATE_NONE;
  815                 kr_barrier(kr);
  816                 kd->kd_kr = NULL;
  817                 kr->kr_kd = NULL;
  818         }
  819 }
  820 
  821 void
  822 kr_free(struct kcov_remote *kr)
  823 {
  824         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  825 
  826         kr->kr_state = KCOV_STATE_DYING;
  827         kr_barrier(kr);
  828         if (kr->kr_kd != NULL)
  829                 kr->kr_kd->kd_kr = NULL;
  830         kr->kr_kd = NULL;
  831         TAILQ_REMOVE(&kr_list, kr, kr_entry);
  832         /* Notify thread(s) waiting in kcov_remote_register(). */
  833         wakeup(kr);
  834         pool_put(&kr_pool, kr);
  835 }
  836 
  837 void
  838 kr_barrier(struct kcov_remote *kr)
  839 {
  840         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  841 
  842         while (kr->kr_nsections > 0)
  843                 msleep_nsec(kr, &kcov_mtx, PWAIT, "kcovbar", INFSLP);
  844 }
  845 
  846 struct kcov_remote *
  847 kr_lookup(int subsystem, void *id)
  848 {
  849         struct kcov_remote *kr;
  850 
  851         MUTEX_ASSERT_LOCKED(&kcov_mtx);
  852 
  853         TAILQ_FOREACH(kr, &kr_list, kr_entry) {
  854                 if (kr->kr_subsystem == subsystem && kr->kr_id == id)
  855                         return (kr);
  856         }
  857         return (NULL);
  858 }
Cache object: d4783cf1eabaa0a153f1d0058477bb85
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/kcov.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/kcov.c
