kern_rmlock.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * Machine independent bits of reader/writer lock implementation.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/11.2/sys/kern/kern_rmlock.c 331722 2018-03-29 02:50:57Z eadler $");
   36 
   37 #include "opt_ddb.h"
   38 
   39 #include <sys/param.h>
   40 #include <sys/systm.h>
   41 
   42 #include <sys/kernel.h>
   43 #include <sys/kdb.h>
   44 #include <sys/ktr.h>
   45 #include <sys/lock.h>
   46 #include <sys/mutex.h>
   47 #include <sys/proc.h>
   48 #include <sys/rmlock.h>
   49 #include <sys/sched.h>
   50 #include <sys/smp.h>
   51 #include <sys/turnstile.h>
   52 #include <sys/lock_profile.h>
   53 #include <machine/cpu.h>
   54 
   55 #ifdef DDB
   56 #include <ddb/ddb.h>
   57 #endif
   58 
   59 /*
   60  * A cookie to mark destroyed rmlocks.  This is stored in the head of
   61  * rm_activeReaders.
   62  */
   63 #define RM_DESTROYED    ((void *)0xdead)
   64 
   65 #define rm_destroyed(rm)                                                \
   66         (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
   67 
   68 #define RMPF_ONQUEUE    1
   69 #define RMPF_SIGNAL     2
   70 
   71 #ifndef INVARIANTS
   72 #define _rm_assert(c, what, file, line)
   73 #endif
   74 
   75 static void     assert_rm(const struct lock_object *lock, int what);
   76 #ifdef DDB
   77 static void     db_show_rm(const struct lock_object *lock);
   78 #endif
   79 static void     lock_rm(struct lock_object *lock, uintptr_t how);
   80 #ifdef KDTRACE_HOOKS
   81 static int      owner_rm(const struct lock_object *lock, struct thread **owner);
   82 #endif
   83 static uintptr_t unlock_rm(struct lock_object *lock);
   84 
   85 struct lock_class lock_class_rm = {
   86         .lc_name = "rm",
   87         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
   88         .lc_assert = assert_rm,
   89 #ifdef DDB
   90         .lc_ddb_show = db_show_rm,
   91 #endif
   92         .lc_lock = lock_rm,
   93         .lc_unlock = unlock_rm,
   94 #ifdef KDTRACE_HOOKS
   95         .lc_owner = owner_rm,
   96 #endif
   97 };
   98 
   99 struct lock_class lock_class_rm_sleepable = {
  100         .lc_name = "sleepable rm",
  101         .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE,
  102         .lc_assert = assert_rm,
  103 #ifdef DDB
  104         .lc_ddb_show = db_show_rm,
  105 #endif
  106         .lc_lock = lock_rm,
  107         .lc_unlock = unlock_rm,
  108 #ifdef KDTRACE_HOOKS
  109         .lc_owner = owner_rm,
  110 #endif
  111 };
  112 
  113 static void
  114 assert_rm(const struct lock_object *lock, int what)
  115 {
  116 
  117         rm_assert((const struct rmlock *)lock, what);
  118 }
  119 
  120 static void
  121 lock_rm(struct lock_object *lock, uintptr_t how)
  122 {
  123         struct rmlock *rm;
  124         struct rm_priotracker *tracker;
  125 
  126         rm = (struct rmlock *)lock;
  127         if (how == 0)
  128                 rm_wlock(rm);
  129         else {
  130                 tracker = (struct rm_priotracker *)how;
  131                 rm_rlock(rm, tracker);
  132         }
  133 }
  134 
  135 static uintptr_t
  136 unlock_rm(struct lock_object *lock)
  137 {
  138         struct thread *td;
  139         struct pcpu *pc;
  140         struct rmlock *rm;
  141         struct rm_queue *queue;
  142         struct rm_priotracker *tracker;
  143         uintptr_t how;
  144 
  145         rm = (struct rmlock *)lock;
  146         tracker = NULL;
  147         how = 0;
  148         rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
  149         if (rm_wowned(rm))
  150                 rm_wunlock(rm);
  151         else {
  152                 /*
  153                  * Find the right rm_priotracker structure for curthread.
  154                  * The guarantee about its uniqueness is given by the fact
  155                  * we already asserted the lock wasn't recursively acquired.
  156                  */
  157                 critical_enter();
  158                 td = curthread;
  159                 pc = pcpu_find(curcpu);
  160                 for (queue = pc->pc_rm_queue.rmq_next;
  161                     queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
  162                         tracker = (struct rm_priotracker *)queue;
  163                                 if ((tracker->rmp_rmlock == rm) &&
  164                                     (tracker->rmp_thread == td)) {
  165                                         how = (uintptr_t)tracker;
  166                                         break;
  167                                 }
  168                 }
  169                 KASSERT(tracker != NULL,
  170                     ("rm_priotracker is non-NULL when lock held in read mode"));
  171                 critical_exit();
  172                 rm_runlock(rm, tracker);
  173         }
  174         return (how);
  175 }
  176 
  177 #ifdef KDTRACE_HOOKS
  178 static int
  179 owner_rm(const struct lock_object *lock, struct thread **owner)
  180 {
  181         const struct rmlock *rm;
  182         struct lock_class *lc;
  183 
  184         rm = (const struct rmlock *)lock;
  185         lc = LOCK_CLASS(&rm->rm_wlock_object);
  186         return (lc->lc_owner(&rm->rm_wlock_object, owner));
  187 }
  188 #endif
  189 
  190 static struct mtx rm_spinlock;
  191 
  192 MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
  193 
  194 /*
  195  * Add or remove tracker from per-cpu list.
  196  *
  197  * The per-cpu list can be traversed at any time in forward direction from an
  198  * interrupt on the *local* cpu.
  199  */
  200 static void inline
  201 rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
  202 {
  203         struct rm_queue *next;
  204 
  205         /* Initialize all tracker pointers */
  206         tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
  207         next = pc->pc_rm_queue.rmq_next;
  208         tracker->rmp_cpuQueue.rmq_next = next;
  209 
  210         /* rmq_prev is not used during froward traversal. */
  211         next->rmq_prev = &tracker->rmp_cpuQueue;
  212 
  213         /* Update pointer to first element. */
  214         pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
  215 }
  216 
  217 /*
  218  * Return a count of the number of trackers the thread 'td' already
  219  * has on this CPU for the lock 'rm'.
  220  */
  221 static int
  222 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
  223     const struct thread *td)
  224 {
  225         struct rm_queue *queue;
  226         struct rm_priotracker *tracker;
  227         int count;
  228 
  229         count = 0;
  230         for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
  231             queue = queue->rmq_next) {
  232                 tracker = (struct rm_priotracker *)queue;
  233                 if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
  234                         count++;
  235         }
  236         return (count);
  237 }
  238 
  239 static void inline
  240 rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
  241 {
  242         struct rm_queue *next, *prev;
  243 
  244         next = tracker->rmp_cpuQueue.rmq_next;
  245         prev = tracker->rmp_cpuQueue.rmq_prev;
  246 
  247         /* Not used during forward traversal. */
  248         next->rmq_prev = prev;
  249 
  250         /* Remove from list. */
  251         prev->rmq_next = next;
  252 }
  253 
  254 static void
  255 rm_cleanIPI(void *arg)
  256 {
  257         struct pcpu *pc;
  258         struct rmlock *rm = arg;
  259         struct rm_priotracker *tracker;
  260         struct rm_queue *queue;
  261         pc = pcpu_find(curcpu);
  262 
  263         for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
  264             queue = queue->rmq_next) {
  265                 tracker = (struct rm_priotracker *)queue;
  266                 if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
  267                         tracker->rmp_flags = RMPF_ONQUEUE;
  268                         mtx_lock_spin(&rm_spinlock);
  269                         LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
  270                             rmp_qentry);
  271                         mtx_unlock_spin(&rm_spinlock);
  272                 }
  273         }
  274 }
  275 
  276 void
  277 rm_init_flags(struct rmlock *rm, const char *name, int opts)
  278 {
  279         struct lock_class *lc;
  280         int liflags, xflags;
  281 
  282         liflags = 0;
  283         if (!(opts & RM_NOWITNESS))
  284                 liflags |= LO_WITNESS;
  285         if (opts & RM_RECURSE)
  286                 liflags |= LO_RECURSABLE;
  287         if (opts & RM_NEW)
  288                 liflags |= LO_NEW;
  289         rm->rm_writecpus = all_cpus;
  290         LIST_INIT(&rm->rm_activeReaders);
  291         if (opts & RM_SLEEPABLE) {
  292                 liflags |= LO_SLEEPABLE;
  293                 lc = &lock_class_rm_sleepable;
  294                 xflags = (opts & RM_NEW ? SX_NEW : 0);
  295                 sx_init_flags(&rm->rm_lock_sx, "rmlock_sx",
  296                     xflags | SX_NOWITNESS);
  297         } else {
  298                 lc = &lock_class_rm;
  299                 xflags = (opts & RM_NEW ? MTX_NEW : 0);
  300                 mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx",
  301                     xflags | MTX_NOWITNESS);
  302         }
  303         lock_init(&rm->lock_object, lc, name, NULL, liflags);
  304 }
  305 
  306 void
  307 rm_init(struct rmlock *rm, const char *name)
  308 {
  309 
  310         rm_init_flags(rm, name, 0);
  311 }
  312 
  313 void
  314 rm_destroy(struct rmlock *rm)
  315 {
  316 
  317         rm_assert(rm, RA_UNLOCKED);
  318         LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
  319         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  320                 sx_destroy(&rm->rm_lock_sx);
  321         else
  322                 mtx_destroy(&rm->rm_lock_mtx);
  323         lock_destroy(&rm->lock_object);
  324 }
  325 
  326 int
  327 rm_wowned(const struct rmlock *rm)
  328 {
  329 
  330         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  331                 return (sx_xlocked(&rm->rm_lock_sx));
  332         else
  333                 return (mtx_owned(&rm->rm_lock_mtx));
  334 }
  335 
  336 void
  337 rm_sysinit(void *arg)
  338 {
  339         struct rm_args *args;
  340 
  341         args = arg;
  342         rm_init_flags(args->ra_rm, args->ra_desc, args->ra_flags);
  343 }
  344 
  345 static int
  346 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
  347 {
  348         struct pcpu *pc;
  349 
  350         critical_enter();
  351         pc = pcpu_find(curcpu);
  352 
  353         /* Check if we just need to do a proper critical_exit. */
  354         if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
  355                 critical_exit();
  356                 return (1);
  357         }
  358 
  359         /* Remove our tracker from the per-cpu list. */
  360         rm_tracker_remove(pc, tracker);
  361 
  362         /* Check to see if the IPI granted us the lock after all. */
  363         if (tracker->rmp_flags) {
  364                 /* Just add back tracker - we hold the lock. */
  365                 rm_tracker_add(pc, tracker);
  366                 critical_exit();
  367                 return (1);
  368         }
  369 
  370         /*
  371          * We allow readers to acquire a lock even if a writer is blocked if
  372          * the lock is recursive and the reader already holds the lock.
  373          */
  374         if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  375                 /*
  376                  * Just grant the lock if this thread already has a tracker
  377                  * for this lock on the per-cpu queue.
  378                  */
  379                 if (rm_trackers_present(pc, rm, curthread) != 0) {
  380                         mtx_lock_spin(&rm_spinlock);
  381                         LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
  382                             rmp_qentry);
  383                         tracker->rmp_flags = RMPF_ONQUEUE;
  384                         mtx_unlock_spin(&rm_spinlock);
  385                         rm_tracker_add(pc, tracker);
  386                         critical_exit();
  387                         return (1);
  388                 }
  389         }
  390 
  391         sched_unpin();
  392         critical_exit();
  393 
  394         if (trylock) {
  395                 if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
  396                         if (!sx_try_xlock(&rm->rm_lock_sx))
  397                                 return (0);
  398                 } else {
  399                         if (!mtx_trylock(&rm->rm_lock_mtx))
  400                                 return (0);
  401                 }
  402         } else {
  403                 if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
  404                         THREAD_SLEEPING_OK();
  405                         sx_xlock(&rm->rm_lock_sx);
  406                         THREAD_NO_SLEEPING();
  407                 } else
  408                         mtx_lock(&rm->rm_lock_mtx);
  409         }
  410 
  411         critical_enter();
  412         pc = pcpu_find(curcpu);
  413         CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
  414         rm_tracker_add(pc, tracker);
  415         sched_pin();
  416         critical_exit();
  417 
  418         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  419                 sx_xunlock(&rm->rm_lock_sx);
  420         else
  421                 mtx_unlock(&rm->rm_lock_mtx);
  422 
  423         return (1);
  424 }
  425 
  426 int
  427 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
  428 {
  429         struct thread *td = curthread;
  430         struct pcpu *pc;
  431 
  432         if (SCHEDULER_STOPPED())
  433                 return (1);
  434 
  435         tracker->rmp_flags  = 0;
  436         tracker->rmp_thread = td;
  437         tracker->rmp_rmlock = rm;
  438 
  439         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  440                 THREAD_NO_SLEEPING();
  441 
  442         td->td_critnest++;      /* critical_enter(); */
  443 
  444         __compiler_membar();
  445 
  446         pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
  447 
  448         rm_tracker_add(pc, tracker);
  449 
  450         sched_pin();
  451 
  452         __compiler_membar();
  453 
  454         td->td_critnest--;
  455 
  456         /*
  457          * Fast path to combine two common conditions into a single
  458          * conditional jump.
  459          */
  460         if (0 == (td->td_owepreempt |
  461             CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
  462                 return (1);
  463 
  464         /* We do not have a read token and need to acquire one. */
  465         return _rm_rlock_hard(rm, tracker, trylock);
  466 }
  467 
  468 static void
  469 _rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
  470 {
  471 
  472         if (td->td_owepreempt) {
  473                 td->td_critnest++;
  474                 critical_exit();
  475         }
  476 
  477         if (!tracker->rmp_flags)
  478                 return;
  479 
  480         mtx_lock_spin(&rm_spinlock);
  481         LIST_REMOVE(tracker, rmp_qentry);
  482 
  483         if (tracker->rmp_flags & RMPF_SIGNAL) {
  484                 struct rmlock *rm;
  485                 struct turnstile *ts;
  486 
  487                 rm = tracker->rmp_rmlock;
  488 
  489                 turnstile_chain_lock(&rm->lock_object);
  490                 mtx_unlock_spin(&rm_spinlock);
  491 
  492                 ts = turnstile_lookup(&rm->lock_object);
  493 
  494                 turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
  495                 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  496                 turnstile_chain_unlock(&rm->lock_object);
  497         } else
  498                 mtx_unlock_spin(&rm_spinlock);
  499 }
  500 
  501 void
  502 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
  503 {
  504         struct pcpu *pc;
  505         struct thread *td = tracker->rmp_thread;
  506 
  507         if (SCHEDULER_STOPPED())
  508                 return;
  509 
  510         td->td_critnest++;      /* critical_enter(); */
  511         pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
  512         rm_tracker_remove(pc, tracker);
  513         td->td_critnest--;
  514         sched_unpin();
  515 
  516         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  517                 THREAD_SLEEPING_OK();
  518 
  519         if (0 == (td->td_owepreempt | tracker->rmp_flags))
  520                 return;
  521 
  522         _rm_unlock_hard(td, tracker);
  523 }
  524 
  525 void
  526 _rm_wlock(struct rmlock *rm)
  527 {
  528         struct rm_priotracker *prio;
  529         struct turnstile *ts;
  530         cpuset_t readcpus;
  531 
  532         if (SCHEDULER_STOPPED())
  533                 return;
  534 
  535         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  536                 sx_xlock(&rm->rm_lock_sx);
  537         else
  538                 mtx_lock(&rm->rm_lock_mtx);
  539 
  540         if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
  541                 /* Get all read tokens back */
  542                 readcpus = all_cpus;
  543                 CPU_NAND(&readcpus, &rm->rm_writecpus);
  544                 rm->rm_writecpus = all_cpus;
  545 
  546                 /*
  547                  * Assumes rm->rm_writecpus update is visible on other CPUs
  548                  * before rm_cleanIPI is called.
  549                  */
  550 #ifdef SMP
  551                 smp_rendezvous_cpus(readcpus,
  552                     smp_no_rendezvous_barrier,
  553                     rm_cleanIPI,
  554                     smp_no_rendezvous_barrier,
  555                     rm);
  556 
  557 #else
  558                 rm_cleanIPI(rm);
  559 #endif
  560 
  561                 mtx_lock_spin(&rm_spinlock);
  562                 while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
  563                         ts = turnstile_trywait(&rm->lock_object);
  564                         prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
  565                         mtx_unlock_spin(&rm_spinlock);
  566                         turnstile_wait(ts, prio->rmp_thread,
  567                             TS_EXCLUSIVE_QUEUE);
  568                         mtx_lock_spin(&rm_spinlock);
  569                 }
  570                 mtx_unlock_spin(&rm_spinlock);
  571         }
  572 }
  573 
  574 void
  575 _rm_wunlock(struct rmlock *rm)
  576 {
  577 
  578         if (rm->lock_object.lo_flags & LO_SLEEPABLE)
  579                 sx_xunlock(&rm->rm_lock_sx);
  580         else
  581                 mtx_unlock(&rm->rm_lock_mtx);
  582 }
  583 
  584 #if LOCK_DEBUG > 0
  585 
  586 void
  587 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
  588 {
  589 
  590         if (SCHEDULER_STOPPED())
  591                 return;
  592 
  593         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  594             ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d",
  595             curthread, rm->lock_object.lo_name, file, line));
  596         KASSERT(!rm_destroyed(rm),
  597             ("rm_wlock() of destroyed rmlock @ %s:%d", file, line));
  598         _rm_assert(rm, RA_UNLOCKED, file, line);
  599 
  600         WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
  601             file, line, NULL);
  602 
  603         _rm_wlock(rm);
  604 
  605         LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
  606         WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
  607         TD_LOCKS_INC(curthread);
  608 }
  609 
  610 void
  611 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
  612 {
  613 
  614         if (SCHEDULER_STOPPED())
  615                 return;
  616 
  617         KASSERT(!rm_destroyed(rm),
  618             ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line));
  619         _rm_assert(rm, RA_WLOCKED, file, line);
  620         WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
  621         LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
  622         _rm_wunlock(rm);
  623         TD_LOCKS_DEC(curthread);
  624 }
  625 
  626 int
  627 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  628     int trylock, const char *file, int line)
  629 {
  630 
  631         if (SCHEDULER_STOPPED())
  632                 return (1);
  633 
  634 #ifdef INVARIANTS
  635         if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
  636                 critical_enter();
  637                 KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
  638                     curthread) == 0,
  639                     ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
  640                     rm->lock_object.lo_name, file, line));
  641                 critical_exit();
  642         }
  643 #endif
  644         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  645             ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
  646             curthread, rm->lock_object.lo_name, file, line));
  647         KASSERT(!rm_destroyed(rm),
  648             ("rm_rlock() of destroyed rmlock @ %s:%d", file, line));
  649         if (!trylock) {
  650                 KASSERT(!rm_wowned(rm),
  651                     ("rm_rlock: wlock already held for %s @ %s:%d",
  652                     rm->lock_object.lo_name, file, line));
  653                 WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
  654                     NULL);
  655         }
  656 
  657         if (_rm_rlock(rm, tracker, trylock)) {
  658                 if (trylock)
  659                         LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
  660                             line);
  661                 else
  662                         LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
  663                             line);
  664                 WITNESS_LOCK(&rm->lock_object, 0, file, line);
  665                 TD_LOCKS_INC(curthread);
  666                 return (1);
  667         } else if (trylock)
  668                 LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
  669 
  670         return (0);
  671 }
  672 
  673 void
  674 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  675     const char *file, int line)
  676 {
  677 
  678         if (SCHEDULER_STOPPED())
  679                 return;
  680 
  681         KASSERT(!rm_destroyed(rm),
  682             ("rm_runlock() of destroyed rmlock @ %s:%d", file, line));
  683         _rm_assert(rm, RA_RLOCKED, file, line);
  684         WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
  685         LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
  686         _rm_runlock(rm, tracker);
  687         TD_LOCKS_DEC(curthread);
  688 }
  689 
  690 #else
  691 
  692 /*
  693  * Just strip out file and line arguments if no lock debugging is enabled in
  694  * the kernel - we are called from a kernel module.
  695  */
  696 void
  697 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
  698 {
  699 
  700         _rm_wlock(rm);
  701 }
  702 
  703 void
  704 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
  705 {
  706 
  707         _rm_wunlock(rm);
  708 }
  709 
  710 int
  711 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  712     int trylock, const char *file, int line)
  713 {
  714 
  715         return _rm_rlock(rm, tracker, trylock);
  716 }
  717 
  718 void
  719 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
  720     const char *file, int line)
  721 {
  722 
  723         _rm_runlock(rm, tracker);
  724 }
  725 
  726 #endif
  727 
  728 #ifdef INVARIANT_SUPPORT
  729 #ifndef INVARIANTS
  730 #undef _rm_assert
  731 #endif
  732 
  733 /*
  734  * Note that this does not need to use witness_assert() for read lock
  735  * assertions since an exact count of read locks held by this thread
  736  * is computable.
  737  */
  738 void
  739 _rm_assert(const struct rmlock *rm, int what, const char *file, int line)
  740 {
  741         int count;
  742 
  743         if (panicstr != NULL)
  744                 return;
  745         switch (what) {
  746         case RA_LOCKED:
  747         case RA_LOCKED | RA_RECURSED:
  748         case RA_LOCKED | RA_NOTRECURSED:
  749         case RA_RLOCKED:
  750         case RA_RLOCKED | RA_RECURSED:
  751         case RA_RLOCKED | RA_NOTRECURSED:
  752                 /*
  753                  * Handle the write-locked case.  Unlike other
  754                  * primitives, writers can never recurse.
  755                  */
  756                 if (rm_wowned(rm)) {
  757                         if (what & RA_RLOCKED)
  758                                 panic("Lock %s exclusively locked @ %s:%d\n",
  759                                     rm->lock_object.lo_name, file, line);
  760                         if (what & RA_RECURSED)
  761                                 panic("Lock %s not recursed @ %s:%d\n",
  762                                     rm->lock_object.lo_name, file, line);
  763                         break;
  764                 }
  765 
  766                 critical_enter();
  767                 count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
  768                 critical_exit();
  769 
  770                 if (count == 0)
  771                         panic("Lock %s not %slocked @ %s:%d\n",
  772                             rm->lock_object.lo_name, (what & RA_RLOCKED) ?
  773                             "read " : "", file, line);
  774                 if (count > 1) {
  775                         if (what & RA_NOTRECURSED)
  776                                 panic("Lock %s recursed @ %s:%d\n",
  777                                     rm->lock_object.lo_name, file, line);
  778                 } else if (what & RA_RECURSED)
  779                         panic("Lock %s not recursed @ %s:%d\n",
  780                             rm->lock_object.lo_name, file, line);
  781                 break;
  782         case RA_WLOCKED:
  783                 if (!rm_wowned(rm))
  784                         panic("Lock %s not exclusively locked @ %s:%d\n",
  785                             rm->lock_object.lo_name, file, line);
  786                 break;
  787         case RA_UNLOCKED:
  788                 if (rm_wowned(rm))
  789                         panic("Lock %s exclusively locked @ %s:%d\n",
  790                             rm->lock_object.lo_name, file, line);
  791 
  792                 critical_enter();
  793                 count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
  794                 critical_exit();
  795 
  796                 if (count != 0)
  797                         panic("Lock %s read locked @ %s:%d\n",
  798                             rm->lock_object.lo_name, file, line);
  799                 break;
  800         default:
  801                 panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
  802                     line);
  803         }
  804 }
  805 #endif /* INVARIANT_SUPPORT */
  806 
  807 #ifdef DDB
  808 static void
  809 print_tracker(struct rm_priotracker *tr)
  810 {
  811         struct thread *td;
  812 
  813         td = tr->rmp_thread;
  814         db_printf("   thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid,
  815             td->td_proc->p_pid, td->td_name);
  816         if (tr->rmp_flags & RMPF_ONQUEUE) {
  817                 db_printf("ONQUEUE");
  818                 if (tr->rmp_flags & RMPF_SIGNAL)
  819                         db_printf(",SIGNAL");
  820         } else
  821                 db_printf("");
  822         db_printf("}\n");
  823 }
  824 
  825 static void
  826 db_show_rm(const struct lock_object *lock)
  827 {
  828         struct rm_priotracker *tr;
  829         struct rm_queue *queue;
  830         const struct rmlock *rm;
  831         struct lock_class *lc;
  832         struct pcpu *pc;
  833 
  834         rm = (const struct rmlock *)lock;
  835         db_printf(" writecpus: ");
  836         ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
  837         db_printf("\n");
  838         db_printf(" per-CPU readers:\n");
  839         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
  840                 for (queue = pc->pc_rm_queue.rmq_next;
  841                     queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
  842                         tr = (struct rm_priotracker *)queue;
  843                         if (tr->rmp_rmlock == rm)
  844                                 print_tracker(tr);
  845                 }
  846         db_printf(" active readers:\n");
  847         LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
  848                 print_tracker(tr);
  849         lc = LOCK_CLASS(&rm->rm_wlock_object);
  850         db_printf("Backing write-lock (%s):\n", lc->lc_name);
  851         lc->lc_ddb_show(&rm->rm_wlock_object);
  852 }
  853 #endif
Cache object: 381b5f733ec3adda42de38ce2903b039
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_rmlock.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rmlock.c
