kern_rmlock.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * Machine independent bits of reader/writer lock implementation.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/11.0/sys/kern/kern_rmlock.c 298819 2016-04-29 22:15:33Z pfg $");
   36 
   37 #include "opt_ddb.h"
   38 
   39 #include <sys/param.h>
   40 #include <sys/systm.h>
   41 
   42 #include <sys/kernel.h>
   43 #include <sys/kdb.h>
   44 #include <sys/ktr.h>
   45 #include <sys/lock.h>
   46 #include <sys/mutex.h>
   47 #include <sys/proc.h>
   48 #include <sys/rmlock.h>
   49 #include <sys/sched.h>
   50 #include <sys/smp.h>
   51 #include <sys/turnstile.h>
   52 #include <sys/lock_profile.h>
   53 #include <machine/cpu.h>
   54 
   55 #ifdef DDB
   56 #include <ddb/ddb.h>
   57 #endif
   58 
   59 /*
   60  * A cookie to mark destroyed rmlocks.  This is stored in the head of
   61  * rm_activeReaders.
   62  */
   63 #define RM_DESTROYED    ((void *)0xdead)
   64 
   65 #define rm_destroyed(rm)                                                \
   66         (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
   67 
   68 #define RMPF_ONQUEUE    1
   69 #define RMPF_SIGNAL     2
   70 
   71 #ifndef INVARIANTS
   72 #define _rm_assert(c, what, file, line)
   73 #endif
   74 
   75 static void     assert_rm(const struct lock_object *lock, int what);
   76 #ifdef DDB
   77 static void     db_show_rm(const struct lock_object *lock);
   78 #endif
   79 static void     lock_rm(struct lock_object *lock, uintptr_t how);
   80 #ifdef KDTRACE_HOOKS
   81 static int      owner_rm(const struct lock_object *lock, struct thread **owner);
   82 #endif
   83 static uintptr_t unlock_rm(struct lock_object *lock);
   84 
   85 struct lock_class lock_class_rm = {
   86         .lc_name = "rm",
   87         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
   88         .lc_assert = assert_rm,
   89 #ifdef DDB
   90         .lc_ddb_show = db_show_rm,
   91 #endif
   92         .lc_lock = lock_rm,
   93         .lc_unlock = unlock_rm,
   94 #ifdef KDTRACE_HOOKS
   95         .lc_owner = owner_rm,
   96 #endif
   97 };
   98 
   99 struct lock_class lock_class_rm_sleepable = {
  100         .lc_name = "sleepable rm",
  101         .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE,
  102         .lc_assert = assert_rm,
  103 #ifdef DDB
  104         .lc_ddb_show = db_show_rm,
  105 #endif
  106         .lc_lock = lock_rm,
  107         .lc_unlock = unlock_rm,
  108 #ifdef KDTRACE_HOOKS
  109         .lc_owner = owner_rm,
  110 #endif
  111 };
  112 
  113 static void
  114 assert_rm(const struct lock_object *lock, int what)
  115 {
  116 
  117         rm_assert((const struct rmlock *)lock, what);
  118 }
  119 
  120 static void
  121 lock_rm(struct lock_object *lock, uintptr_t how)
  122 {
  123         struct rmlock *rm;
  124         struct rm_priotracker *tracker;
  125 
  126         rm = (struct rmlock *)lock;
  127         if (how == 0)
  128                 rm_wlock(rm);
  129         else {
  130                 tracker = (struct rm_priotracker *)how;
  131                 rm_rlock(rm, tracker);
  132         }
  133 }
  134 
  135 static uintptr_t
  136 unlock_rm(struct lock_object *lock)
  137 {
  138         struct thread *td;
  139         struct pcpu *pc;
  140         struct rmlock *rm;
  141         struct rm_queue *queue;
  142         struct rm_priotracker *tracker;
  143         uintptr_t how;
  144 
  145         rm = (struct rmlock *)lock;
  146         tracker = NULL;
  147         how = 0;
  148         rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
  149         if (rm_wowned(rm))
  150                 rm_wunlock(rm);
  151         else {
  152                 /*
  153                  * Find the right rm_priotracker structure for curthread.
  154                  * The guarantee about its uniqueness is given by the fact
  155                  * we already asserted the lock wasn't recursively acquired.
  156                  */
  157                 critical_enter();
  158                 td = curthread;
  159                 pc = pcpu_find(curcpu);
  160                 for (queue = pc->pc_rm_queue.rmq_next;
  161                     queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
  162                         tracker = (struct rm_priotracker *)queue;
  163                                 if ((tracker->rmp_rmlock == rm) &&
  164                                     (tracker->rmp_thread == td)) {
  165                                         how = (uintptr_t)tracker;
  166                                         break;
  167                                 }
  168                 }
  169                 KASSERT(tracker != NULL,
  170                     ("rm_priotracker is non-NULL when lock held in read mode"));
  171                 critical_exit();
  172                 rm_runlock(rm, tracker);
  173         }
  174         return (how);
  175 }
  176 
  177 #ifdef KDTRACE_HOOKS
  178 static int
  179 owner_rm(const struct lock_object *lock, struct thread **owner)
  180 {
  181         const struct rmlock *rm;
  182         struct lock_class *lc;
  183 
  184         rm = (const struct rmlock *)lock;
  185         lc = LOCK_CLASS(&rm->rm_wlock_object);
  186         return (lc->lc_owner(&rm->rm_wlock_object, owner));
  187 }
  188 #endif
  189 
  190 static struct mtx rm_spinlock;
  191 
  192 MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
  193 
  194 /*
  195  * Add or remove tracker from per-cpu list.
  196  *
  197  * The per-cpu list can be traversed at any time in forward direction from an
  198  * interrupt on the *local* cpu.
  199  */
  200 static void inline
  201 rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
  202 {
  203         struct rm_queue *next;
  204 
  205         /* Initialize all tracker pointers */
  206         tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
  207         next = pc->pc_rm_queue.rmq_next;
  208         tracker->rmp_cpuQueue.rmq_next = next;
  209 
  210         /* rmq_prev is not used during froward traversal. */
  211         next->rmq_prev = &tracker->rmp_cpuQueue;
  212 
  213         /* Update pointer to first element. */
  214         pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
  215 }
  216 
  217 /*
  218  * Return a count of the number of trackers the thread 'td' already
  219  * has on this CPU for the lock 'rm'.
  220  */
  221 static int
  222 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
  223     const struct thread *td)
  224 {
  225         struct rm_queue *queue;
  226         struct rm_priotracker *tracker;
  227         int count;
  228 
  229         count = 0;
  230         for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
  231             queue = queue->rmq_next) {
  232                 tracker = (struct rm_priotracker *)queue;
  233                 if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
  234                         count++;
  235         }
  236         return (count);
  237 }
  238 
  239 static void inline
  240 rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
  241 {
  242         struct rm_queue *next, *prev;
  243 
  244         next = tracker->rmp_cpuQueue.rmq_next;
  245         prev = tracker->rmp_cpuQueue.rmq_prev;
  246 
  247         /* Not used during forward traversal. */
  248         next->rmq_prev = prev;
  249 
  250         /* Remove from list. */
  251         prev->rmq_next = next;
  252 }
  253 
  254 static void
  255 rm_cleanIPI(void *arg)
  256 {
  257         struct pcpu *pc;
  258         struct rmlock *rm = arg;
  259         struct rm_priotracker *tracker;
  260         struct rm_queue *queue;
  261         pc = pcpu_find(curcpu);
  262 
  263         for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
  264             queue = queue->rmq_next) {
  265                 tracker = (struct rm_priotracker *)queue;
  266                 if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
  267                         tracker->rmp_flags = RMPF_ONQUEUE;
  268                         mtx_lock_spin(&rm_spinlock);
  269                         LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
  270                             rmp_qentry);
  271                         mtx_unlock_spin(&rm_spinlock);
  272                 }
  273         }
  274 }
  275 
  276 void
  277 rm_init_flags(struct rmlock *rm, const char *name, int opts)
  278 {
  279         struct lock_class *lc;
  280         int liflags, xflags;
  281 
  282         liflags = 0;
  283         if (!(opts & RM_NOWITNESS))
  284                 liflags |= LO_WITNESS;
  285         if (opts & RM_RECURSE)
  286                 liflags |= LO_RECURSABLE;
  287         if (opts & RM_NEW)
  288                 liflags |= LO_NEW;
  289         rm->rm_writecpus = all_cpus;
  290         LIST_INIT(&rm->rm_activeReaders);
  291         if (opts & RM_SLEEPABLE) {
  292                 liflags |= LO_SLEEPABLE;
  293                 lc = &lock_class_rm_sleepable;
  294                 xflags = (opts & RM_NEW ? SX_NEW : 0);
  295                 sx_init_flags(&rm->rm_lock_sx, "rmlock_sx",
  296                     xflags | SX_NOWITNESS);
  297         } else {
  298                 lc = &lock_class_rm;
  299                 xflags = (opts & RM_NEW ? MTX_NEW : 0);
  300                 mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx",
  301                     xflags | MTX_NOWITNESS);
  302         }
  303         lock_init(&rm->lock_object, lc, name, NULL, liflags);
  304 }
  305 
  306 void
  307 rm_init(struct rmlock *rm, const char *name)
  308 {
  309 
  310         rm_init_flags(rm, name, 0);
  311 }
  312 
  313 void
  314 rm_destroy(struct rmlock *rm)
  315 {
  316 
  317         rm_assert(rm, RA_UNLOCKED);
  318         LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
  319         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  320                 sx_destroy(&rm->rm_lock_sx);
  321         else
  322                 mtx_destroy(&rm->rm_lock_mtx);
  323         lock_destroy(&rm->lock_object);
  324 }
  325 
  326 int
  327 rm_wowned(const struct rmlock *rm)
  328 {
  329 
  330         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  331                 return (sx_xlocked(&rm->rm_lock_sx));
  332         else
  333                 return (mtx_owned(&rm->rm_lock_mtx));
  334 }
  335 
  336 void
  337 rm_sysinit(void *arg)
  338 {
  339         struct rm_args *args = arg;
  340 
  341         rm_init(args->ra_rm, args->ra_desc);
  342 }
  343 
  344 void
  345 rm_sysinit_flags(void *arg)
  346 {
  347         struct rm_args_flags *args = arg;
  348 
  349         rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts);
  350 }
  351 
  352 static int
  353 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
  354 {
  355         struct pcpu *pc;
  356 
  357         critical_enter();
  358         pc = pcpu_find(curcpu);
  359 
  360         /* Check if we just need to do a proper critical_exit. */
  361         if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
  362                 critical_exit();
  363                 return (1);
  364         }
  365 
  366         /* Remove our tracker from the per-cpu list. */
  367         rm_tracker_remove(pc, tracker);
  368 
  369         /* Check to see if the IPI granted us the lock after all. */
  370         if (tracker->rmp_flags) {
  371                 /* Just add back tracker - we hold the lock. */
  372                 rm_tracker_add(pc, tracker);
  373                 critical_exit();
  374                 return (1);
  375         }
  376 
  377         /*
  378          * We allow readers to acquire a lock even if a writer is blocked if
  379          * the lock is recursive and the reader already holds the lock.
  380          */
  381         if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  382                 /*
  383                  * Just grant the lock if this thread already has a tracker
  384                  * for this lock on the per-cpu queue.
  385                  */
  386                 if (rm_trackers_present(pc, rm, curthread) != 0) {
  387                         mtx_lock_spin(&rm_spinlock);
  388                         LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
  389                             rmp_qentry);
  390                         tracker->rmp_flags = RMPF_ONQUEUE;
  391                         mtx_unlock_spin(&rm_spinlock);
  392                         rm_tracker_add(pc, tracker);
  393                         critical_exit();
  394                         return (1);
  395                 }
  396         }
  397 
  398         sched_unpin();
  399         critical_exit();
  400 
  401         if (trylock) {
  402                 if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
  403                         if (!sx_try_xlock(&rm->rm_lock_sx))
  404                                 return (0);
  405                 } else {
  406                         if (!mtx_trylock(&rm->rm_lock_mtx))
  407                                 return (0);
  408                 }
  409         } else {
  410                 if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
  411                         THREAD_SLEEPING_OK();
  412                         sx_xlock(&rm->rm_lock_sx);
  413                         THREAD_NO_SLEEPING();
  414                 } else
  415                         mtx_lock(&rm->rm_lock_mtx);
  416         }
  417 
  418         critical_enter();
  419         pc = pcpu_find(curcpu);
  420         CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
  421         rm_tracker_add(pc, tracker);
  422         sched_pin();
  423         critical_exit();
  424 
  425         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  426                 sx_xunlock(&rm->rm_lock_sx);
  427         else
  428                 mtx_unlock(&rm->rm_lock_mtx);
  429 
  430         return (1);
  431 }
  432 
  433 int
  434 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
  435 {
  436         struct thread *td = curthread;
  437         struct pcpu *pc;
  438 
  439         if (SCHEDULER_STOPPED())
  440                 return (1);
  441 
  442         tracker->rmp_flags  = 0;
  443         tracker->rmp_thread = td;
  444         tracker->rmp_rmlock = rm;
  445 
  446         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  447                 THREAD_NO_SLEEPING();
  448 
  449         td->td_critnest++;      /* critical_enter(); */
  450 
  451         __compiler_membar();
  452 
  453         pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
  454 
  455         rm_tracker_add(pc, tracker);
  456 
  457         sched_pin();
  458 
  459         __compiler_membar();
  460 
  461         td->td_critnest--;
  462 
  463         /*
  464          * Fast path to combine two common conditions into a single
  465          * conditional jump.
  466          */
  467         if (0 == (td->td_owepreempt |
  468             CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
  469                 return (1);
  470 
  471         /* We do not have a read token and need to acquire one. */
  472         return _rm_rlock_hard(rm, tracker, trylock);
  473 }
  474 
  475 static void
  476 _rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
  477 {
  478 
  479         if (td->td_owepreempt) {
  480                 td->td_critnest++;
  481                 critical_exit();
  482         }
  483 
  484         if (!tracker->rmp_flags)
  485                 return;
  486 
  487         mtx_lock_spin(&rm_spinlock);
  488         LIST_REMOVE(tracker, rmp_qentry);
  489 
  490         if (tracker->rmp_flags & RMPF_SIGNAL) {
  491                 struct rmlock *rm;
  492                 struct turnstile *ts;
  493 
  494                 rm = tracker->rmp_rmlock;
  495 
  496                 turnstile_chain_lock(&rm->lock_object);
  497                 mtx_unlock_spin(&rm_spinlock);
  498 
  499                 ts = turnstile_lookup(&rm->lock_object);
  500 
  501                 turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
  502                 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  503                 turnstile_chain_unlock(&rm->lock_object);
  504         } else
  505                 mtx_unlock_spin(&rm_spinlock);
  506 }
  507 
  508 void
  509 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
  510 {
  511         struct pcpu *pc;
  512         struct thread *td = tracker->rmp_thread;
  513 
  514         if (SCHEDULER_STOPPED())
  515                 return;
  516 
  517         td->td_critnest++;      /* critical_enter(); */
  518         pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
  519         rm_tracker_remove(pc, tracker);
  520         td->td_critnest--;
  521         sched_unpin();
  522 
  523         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  524                 THREAD_SLEEPING_OK();
  525 
  526         if (0 == (td->td_owepreempt | tracker->rmp_flags))
  527                 return;
  528 
  529         _rm_unlock_hard(td, tracker);
  530 }
  531 
  532 void
  533 _rm_wlock(struct rmlock *rm)
  534 {
  535         struct rm_priotracker *prio;
  536         struct turnstile *ts;
  537         cpuset_t readcpus;
  538 
  539         if (SCHEDULER_STOPPED())
  540                 return;
  541 
  542         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  543                 sx_xlock(&rm->rm_lock_sx);
  544         else
  545                 mtx_lock(&rm->rm_lock_mtx);
  546 
  547         if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
  548                 /* Get all read tokens back */
  549                 readcpus = all_cpus;
  550                 CPU_NAND(&readcpus, &rm->rm_writecpus);
  551                 rm->rm_writecpus = all_cpus;
  552 
  553                 /*
  554                  * Assumes rm->rm_writecpus update is visible on other CPUs
  555                  * before rm_cleanIPI is called.
  556                  */
  557 #ifdef SMP
  558                 smp_rendezvous_cpus(readcpus,
  559                     smp_no_rendevous_barrier,
  560                     rm_cleanIPI,
  561                     smp_no_rendevous_barrier,
  562                     rm);
  563 
  564 #else
  565                 rm_cleanIPI(rm);
  566 #endif
  567 
  568                 mtx_lock_spin(&rm_spinlock);
  569                 while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
  570                         ts = turnstile_trywait(&rm->lock_object);
  571                         prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
  572                         mtx_unlock_spin(&rm_spinlock);
  573                         turnstile_wait(ts, prio->rmp_thread,
  574                             TS_EXCLUSIVE_QUEUE);
  575                         mtx_lock_spin(&rm_spinlock);
  576                 }
  577                 mtx_unlock_spin(&rm_spinlock);
  578         }
  579 }
  580 
  581 void
  582 _rm_wunlock(struct rmlock *rm)
  583 {
  584 
  585         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  586                 sx_xunlock(&rm->rm_lock_sx);
  587         else
  588                 mtx_unlock(&rm->rm_lock_mtx);
  589 }
  590 
  591 #if LOCK_DEBUG > 0
  592 
  593 void
  594 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
  595 {
  596 
  597         if (SCHEDULER_STOPPED())
  598                 return;
  599 
  600         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  601             ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d",
  602             curthread, rm->lock_object.lo_name, file, line));
  603         KASSERT(!rm_destroyed(rm),
  604             ("rm_wlock() of destroyed rmlock @ %s:%d", file, line));
  605         _rm_assert(rm, RA_UNLOCKED, file, line);
  606 
  607         WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
  608             file, line, NULL);
  609 
  610         _rm_wlock(rm);
  611 
  612         LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
  613         WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
  614         TD_LOCKS_INC(curthread);
  615 }
  616 
  617 void
  618 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
  619 {
  620 
  621         if (SCHEDULER_STOPPED())
  622                 return;
  623 
  624         KASSERT(!rm_destroyed(rm),
  625             ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line));
  626         _rm_assert(rm, RA_WLOCKED, file, line);
  627         WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
  628         LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
  629         _rm_wunlock(rm);
  630         TD_LOCKS_DEC(curthread);
  631 }
  632 
  633 int
  634 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  635     int trylock, const char *file, int line)
  636 {
  637 
  638         if (SCHEDULER_STOPPED())
  639                 return (1);
  640 
  641 #ifdef INVARIANTS
  642         if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
  643                 critical_enter();
  644                 KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
  645                     curthread) == 0,
  646                     ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
  647                     rm->lock_object.lo_name, file, line));
  648                 critical_exit();
  649         }
  650 #endif
  651         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  652             ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
  653             curthread, rm->lock_object.lo_name, file, line));
  654         KASSERT(!rm_destroyed(rm),
  655             ("rm_rlock() of destroyed rmlock @ %s:%d", file, line));
  656         if (!trylock) {
  657                 KASSERT(!rm_wowned(rm),
  658                     ("rm_rlock: wlock already held for %s @ %s:%d",
  659                     rm->lock_object.lo_name, file, line));
  660                 WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
  661                     NULL);
  662         }
  663 
  664         if (_rm_rlock(rm, tracker, trylock)) {
  665                 if (trylock)
  666                         LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
  667                             line);
  668                 else
  669                         LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
  670                             line);
  671                 WITNESS_LOCK(&rm->lock_object, 0, file, line);
  672                 TD_LOCKS_INC(curthread);
  673                 return (1);
  674         } else if (trylock)
  675                 LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
  676 
  677         return (0);
  678 }
  679 
  680 void
  681 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  682     const char *file, int line)
  683 {
  684 
  685         if (SCHEDULER_STOPPED())
  686                 return;
  687 
  688         KASSERT(!rm_destroyed(rm),
  689             ("rm_runlock() of destroyed rmlock @ %s:%d", file, line));
  690         _rm_assert(rm, RA_RLOCKED, file, line);
  691         WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
  692         LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
  693         _rm_runlock(rm, tracker);
  694         TD_LOCKS_DEC(curthread);
  695 }
  696 
  697 #else
  698 
  699 /*
  700  * Just strip out file and line arguments if no lock debugging is enabled in
  701  * the kernel - we are called from a kernel module.
  702  */
  703 void
  704 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
  705 {
  706 
  707         _rm_wlock(rm);
  708 }
  709 
  710 void
  711 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
  712 {
  713 
  714         _rm_wunlock(rm);
  715 }
  716 
  717 int
  718 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  719     int trylock, const char *file, int line)
  720 {
  721 
  722         return _rm_rlock(rm, tracker, trylock);
  723 }
  724 
  725 void
  726 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  727     const char *file, int line)
  728 {
  729 
  730         _rm_runlock(rm, tracker);
  731 }
  732 
  733 #endif
  734 
  735 #ifdef INVARIANT_SUPPORT
  736 #ifndef INVARIANTS
  737 #undef _rm_assert
  738 #endif
  739 
  740 /*
  741  * Note that this does not need to use witness_assert() for read lock
  742  * assertions since an exact count of read locks held by this thread
  743  * is computable.
  744  */
  745 void
  746 _rm_assert(const struct rmlock *rm, int what, const char *file, int line)
  747 {
  748         int count;
  749 
  750         if (panicstr != NULL)
  751                 return;
  752         switch (what) {
  753         case RA_LOCKED:
  754         case RA_LOCKED | RA_RECURSED:
  755         case RA_LOCKED | RA_NOTRECURSED:
  756         case RA_RLOCKED:
  757         case RA_RLOCKED | RA_RECURSED:
  758         case RA_RLOCKED | RA_NOTRECURSED:
  759                 /*
  760                  * Handle the write-locked case.  Unlike other
  761                  * primitives, writers can never recurse.
  762                  */
  763                 if (rm_wowned(rm)) {
  764                         if (what & RA_RLOCKED)
  765                                 panic("Lock %s exclusively locked @ %s:%d\n",
  766                                     rm->lock_object.lo_name, file, line);
  767                         if (what & RA_RECURSED)
  768                                 panic("Lock %s not recursed @ %s:%d\n",
  769                                     rm->lock_object.lo_name, file, line);
  770                         break;
  771                 }
  772 
  773                 critical_enter();
  774                 count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
  775                 critical_exit();
  776 
  777                 if (count == 0)
  778                         panic("Lock %s not %slocked @ %s:%d\n",
  779                             rm->lock_object.lo_name, (what & RA_RLOCKED) ?
  780                             "read " : "", file, line);
  781                 if (count > 1) {
  782                         if (what & RA_NOTRECURSED)
  783                                 panic("Lock %s recursed @ %s:%d\n",
  784                                     rm->lock_object.lo_name, file, line);
  785                 } else if (what & RA_RECURSED)
  786                         panic("Lock %s not recursed @ %s:%d\n",
  787                             rm->lock_object.lo_name, file, line);
  788                 break;
  789         case RA_WLOCKED:
  790                 if (!rm_wowned(rm))
  791                         panic("Lock %s not exclusively locked @ %s:%d\n",
  792                             rm->lock_object.lo_name, file, line);
  793                 break;
  794         case RA_UNLOCKED:
  795                 if (rm_wowned(rm))
  796                         panic("Lock %s exclusively locked @ %s:%d\n",
  797                             rm->lock_object.lo_name, file, line);
  798 
  799                 critical_enter();
  800                 count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
  801                 critical_exit();
  802 
  803                 if (count != 0)
  804                         panic("Lock %s read locked @ %s:%d\n",
  805                             rm->lock_object.lo_name, file, line);
  806                 break;
  807         default:
  808                 panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
  809                     line);
  810         }
  811 }
  812 #endif /* INVARIANT_SUPPORT */
  813 
  814 #ifdef DDB
  815 static void
  816 print_tracker(struct rm_priotracker *tr)
  817 {
  818         struct thread *td;
  819 
  820         td = tr->rmp_thread;
  821         db_printf("   thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid,
  822             td->td_proc->p_pid, td->td_name);
  823         if (tr->rmp_flags & RMPF_ONQUEUE) {
  824                 db_printf("ONQUEUE");
  825                 if (tr->rmp_flags & RMPF_SIGNAL)
  826                         db_printf(",SIGNAL");
  827         } else
  828                 db_printf("");
  829         db_printf("}\n");
  830 }
  831 
  832 static void
  833 db_show_rm(const struct lock_object *lock)
  834 {
  835         struct rm_priotracker *tr;
  836         struct rm_queue *queue;
  837         const struct rmlock *rm;
  838         struct lock_class *lc;
  839         struct pcpu *pc;
  840 
  841         rm = (const struct rmlock *)lock;
  842         db_printf(" writecpus: ");
  843         ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
  844         db_printf("\n");
  845         db_printf(" per-CPU readers:\n");
  846         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
  847                 for (queue = pc->pc_rm_queue.rmq_next;
  848                     queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
  849                         tr = (struct rm_priotracker *)queue;
  850                         if (tr->rmp_rmlock == rm)
  851                                 print_tracker(tr);
  852                 }
  853         db_printf(" active readers:\n");
  854         LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
  855                 print_tracker(tr);
  856         lc = LOCK_CLASS(&rm->rm_wlock_object);
  857         db_printf("Backing write-lock (%s):\n", lc->lc_name);
  858         lc->lc_ddb_show(&rm->rm_wlock_object);
  859 }
  860 #endif
Cache object: 89f7b8ce81743cfd0cc134768c91504c
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_rmlock.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rmlock.c
