The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rwlock.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 /*
   28  * Machine independent bits of reader/writer lock implementation.
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD: releng/11.0/sys/kern/kern_rwlock.c 303953 2016-08-11 09:28:49Z mjg $");
   33 
   34 #include "opt_ddb.h"
   35 #include "opt_hwpmc_hooks.h"
   36 #include "opt_no_adaptive_rwlocks.h"
   37 
   38 #include <sys/param.h>
   39 #include <sys/kdb.h>
   40 #include <sys/ktr.h>
   41 #include <sys/kernel.h>
   42 #include <sys/lock.h>
   43 #include <sys/mutex.h>
   44 #include <sys/proc.h>
   45 #include <sys/rwlock.h>
   46 #include <sys/sched.h>
   47 #include <sys/smp.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/systm.h>
   50 #include <sys/turnstile.h>
   51 
   52 #include <machine/cpu.h>
   53 
   54 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
   55 #define ADAPTIVE_RWLOCKS
   56 #endif
   57 
   58 #ifdef HWPMC_HOOKS
   59 #include <sys/pmckern.h>
   60 PMC_SOFT_DECLARE( , , lock, failed);
   61 #endif
   62 
   63 /*
   64  * Return the rwlock address when the lock cookie address is provided.
   65  * This functionality assumes that struct rwlock* have a member named rw_lock.
   66  */
   67 #define rwlock2rw(c)    (__containerof(c, struct rwlock, rw_lock))
   68 
   69 #ifdef DDB
   70 #include <ddb/ddb.h>
   71 
   72 static void     db_show_rwlock(const struct lock_object *lock);
   73 #endif
   74 static void     assert_rw(const struct lock_object *lock, int what);
   75 static void     lock_rw(struct lock_object *lock, uintptr_t how);
   76 #ifdef KDTRACE_HOOKS
   77 static int      owner_rw(const struct lock_object *lock, struct thread **owner);
   78 #endif
   79 static uintptr_t unlock_rw(struct lock_object *lock);
   80 
   81 struct lock_class lock_class_rw = {
   82         .lc_name = "rw",
   83         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE,
   84         .lc_assert = assert_rw,
   85 #ifdef DDB
   86         .lc_ddb_show = db_show_rwlock,
   87 #endif
   88         .lc_lock = lock_rw,
   89         .lc_unlock = unlock_rw,
   90 #ifdef KDTRACE_HOOKS
   91         .lc_owner = owner_rw,
   92 #endif
   93 };
   94 
   95 #ifdef ADAPTIVE_RWLOCKS
   96 static int rowner_retries = 10;
   97 static int rowner_loops = 10000;
   98 static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL,
   99     "rwlock debugging");
  100 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
  101 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
  102 
  103 static struct lock_delay_config rw_delay = {
  104         .initial        = 1000,
  105         .step           = 500,
  106         .min            = 100,
  107         .max            = 5000,
  108 };
  109 
  110 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_initial, CTLFLAG_RW, &rw_delay.initial,
  111     0, "");
  112 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_step, CTLFLAG_RW, &rw_delay.step,
  113     0, "");
  114 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_min, CTLFLAG_RW, &rw_delay.min,
  115     0, "");
  116 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max,
  117     0, "");
  118 
  119 static void
  120 rw_delay_sysinit(void *dummy)
  121 {
  122 
  123         rw_delay.initial = mp_ncpus * 25;
  124         rw_delay.step = (mp_ncpus * 25) / 2;
  125         rw_delay.min = mp_ncpus * 5;
  126         rw_delay.max = mp_ncpus * 25 * 10;
  127 }
  128 LOCK_DELAY_SYSINIT(rw_delay_sysinit);
  129 #endif
  130 
  131 /*
  132  * Return a pointer to the owning thread if the lock is write-locked or
  133  * NULL if the lock is unlocked or read-locked.
  134  */
  135 #define rw_wowner(rw)                                                   \
  136         ((rw)->rw_lock & RW_LOCK_READ ? NULL :                          \
  137             (struct thread *)RW_OWNER((rw)->rw_lock))
  138 
  139 /*
  140  * Returns if a write owner is recursed.  Write ownership is not assured
  141  * here and should be previously checked.
  142  */
  143 #define rw_recursed(rw)         ((rw)->rw_recurse != 0)
  144 
  145 /*
  146  * Return true if curthread helds the lock.
  147  */
  148 #define rw_wlocked(rw)          (rw_wowner((rw)) == curthread)
  149 
  150 /*
  151  * Return a pointer to the owning thread for this lock who should receive
  152  * any priority lent by threads that block on this lock.  Currently this
  153  * is identical to rw_wowner().
  154  */
  155 #define rw_owner(rw)            rw_wowner(rw)
  156 
  157 #ifndef INVARIANTS
  158 #define __rw_assert(c, what, file, line)
  159 #endif
  160 
  161 void
  162 assert_rw(const struct lock_object *lock, int what)
  163 {
  164 
  165         rw_assert((const struct rwlock *)lock, what);
  166 }
  167 
  168 void
  169 lock_rw(struct lock_object *lock, uintptr_t how)
  170 {
  171         struct rwlock *rw;
  172 
  173         rw = (struct rwlock *)lock;
  174         if (how)
  175                 rw_rlock(rw);
  176         else
  177                 rw_wlock(rw);
  178 }
  179 
  180 uintptr_t
  181 unlock_rw(struct lock_object *lock)
  182 {
  183         struct rwlock *rw;
  184 
  185         rw = (struct rwlock *)lock;
  186         rw_assert(rw, RA_LOCKED | LA_NOTRECURSED);
  187         if (rw->rw_lock & RW_LOCK_READ) {
  188                 rw_runlock(rw);
  189                 return (1);
  190         } else {
  191                 rw_wunlock(rw);
  192                 return (0);
  193         }
  194 }
  195 
  196 #ifdef KDTRACE_HOOKS
  197 int
  198 owner_rw(const struct lock_object *lock, struct thread **owner)
  199 {
  200         const struct rwlock *rw = (const struct rwlock *)lock;
  201         uintptr_t x = rw->rw_lock;
  202 
  203         *owner = rw_wowner(rw);
  204         return ((x & RW_LOCK_READ) != 0 ?  (RW_READERS(x) != 0) :
  205             (*owner != NULL));
  206 }
  207 #endif
  208 
  209 void
  210 _rw_init_flags(volatile uintptr_t *c, const char *name, int opts)
  211 {
  212         struct rwlock *rw;
  213         int flags;
  214 
  215         rw = rwlock2rw(c);
  216 
  217         MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET |
  218             RW_RECURSE | RW_NEW)) == 0);
  219         ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock,
  220             ("%s: rw_lock not aligned for %s: %p", __func__, name,
  221             &rw->rw_lock));
  222 
  223         flags = LO_UPGRADABLE;
  224         if (opts & RW_DUPOK)
  225                 flags |= LO_DUPOK;
  226         if (opts & RW_NOPROFILE)
  227                 flags |= LO_NOPROFILE;
  228         if (!(opts & RW_NOWITNESS))
  229                 flags |= LO_WITNESS;
  230         if (opts & RW_RECURSE)
  231                 flags |= LO_RECURSABLE;
  232         if (opts & RW_QUIET)
  233                 flags |= LO_QUIET;
  234         if (opts & RW_NEW)
  235                 flags |= LO_NEW;
  236 
  237         lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
  238         rw->rw_lock = RW_UNLOCKED;
  239         rw->rw_recurse = 0;
  240 }
  241 
  242 void
  243 _rw_destroy(volatile uintptr_t *c)
  244 {
  245         struct rwlock *rw;
  246 
  247         rw = rwlock2rw(c);
  248 
  249         KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw));
  250         KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw));
  251         rw->rw_lock = RW_DESTROYED;
  252         lock_destroy(&rw->lock_object);
  253 }
  254 
  255 void
  256 rw_sysinit(void *arg)
  257 {
  258         struct rw_args *args = arg;
  259 
  260         rw_init((struct rwlock *)args->ra_rw, args->ra_desc);
  261 }
  262 
  263 void
  264 rw_sysinit_flags(void *arg)
  265 {
  266         struct rw_args_flags *args = arg;
  267 
  268         rw_init_flags((struct rwlock *)args->ra_rw, args->ra_desc,
  269             args->ra_flags);
  270 }
  271 
  272 int
  273 _rw_wowned(const volatile uintptr_t *c)
  274 {
  275 
  276         return (rw_wowner(rwlock2rw(c)) == curthread);
  277 }
  278 
  279 void
  280 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
  281 {
  282         struct rwlock *rw;
  283 
  284         if (SCHEDULER_STOPPED())
  285                 return;
  286 
  287         rw = rwlock2rw(c);
  288 
  289         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  290             ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d",
  291             curthread, rw->lock_object.lo_name, file, line));
  292         KASSERT(rw->rw_lock != RW_DESTROYED,
  293             ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
  294         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
  295             line, NULL);
  296         __rw_wlock(rw, curthread, file, line);
  297         LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
  298         WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
  299         TD_LOCKS_INC(curthread);
  300 }
  301 
  302 int
  303 __rw_try_wlock(volatile uintptr_t *c, const char *file, int line)
  304 {
  305         struct rwlock *rw;
  306         int rval;
  307 
  308         if (SCHEDULER_STOPPED())
  309                 return (1);
  310 
  311         rw = rwlock2rw(c);
  312 
  313         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  314             ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d",
  315             curthread, rw->lock_object.lo_name, file, line));
  316         KASSERT(rw->rw_lock != RW_DESTROYED,
  317             ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
  318 
  319         if (rw_wlocked(rw) &&
  320             (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  321                 rw->rw_recurse++;
  322                 rval = 1;
  323         } else
  324                 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
  325                     (uintptr_t)curthread);
  326 
  327         LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
  328         if (rval) {
  329                 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
  330                     file, line);
  331                 if (!rw_recursed(rw))
  332                         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
  333                             rw, 0, 0, file, line, LOCKSTAT_WRITER);
  334                 TD_LOCKS_INC(curthread);
  335         }
  336         return (rval);
  337 }
  338 
  339 void
  340 _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
  341 {
  342         struct rwlock *rw;
  343 
  344         if (SCHEDULER_STOPPED())
  345                 return;
  346 
  347         rw = rwlock2rw(c);
  348 
  349         KASSERT(rw->rw_lock != RW_DESTROYED,
  350             ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
  351         __rw_assert(c, RA_WLOCKED, file, line);
  352         WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
  353         LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
  354             line);
  355         __rw_wunlock(rw, curthread, file, line);
  356         TD_LOCKS_DEC(curthread);
  357 }
  358 
  359 /*
  360  * Determines whether a new reader can acquire a lock.  Succeeds if the
  361  * reader already owns a read lock and the lock is locked for read to
  362  * prevent deadlock from reader recursion.  Also succeeds if the lock
  363  * is unlocked and has no writer waiters or spinners.  Failing otherwise
  364  * prioritizes writers before readers.
  365  */
  366 #define RW_CAN_READ(_rw)                                                \
  367     ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) &      \
  368     (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==  \
  369     RW_LOCK_READ)
  370 
  371 void
  372 __rw_rlock(volatile uintptr_t *c, const char *file, int line)
  373 {
  374         struct rwlock *rw;
  375         struct turnstile *ts;
  376 #ifdef ADAPTIVE_RWLOCKS
  377         volatile struct thread *owner;
  378         int spintries = 0;
  379         int i;
  380 #endif
  381 #ifdef LOCK_PROFILING
  382         uint64_t waittime = 0;
  383         int contested = 0;
  384 #endif
  385         uintptr_t v;
  386 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
  387         struct lock_delay_arg lda;
  388 #endif
  389 #ifdef KDTRACE_HOOKS
  390         uintptr_t state;
  391         u_int sleep_cnt = 0;
  392         int64_t sleep_time = 0;
  393         int64_t all_time = 0;
  394 #endif
  395 
  396         if (SCHEDULER_STOPPED())
  397                 return;
  398 
  399 #if defined(ADAPTIVE_RWLOCKS)
  400         lock_delay_arg_init(&lda, &rw_delay);
  401 #elif defined(KDTRACE_HOOKS)
  402         lock_delay_arg_init(&lda, NULL);
  403 #endif
  404         rw = rwlock2rw(c);
  405 
  406         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  407             ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
  408             curthread, rw->lock_object.lo_name, file, line));
  409         KASSERT(rw->rw_lock != RW_DESTROYED,
  410             ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
  411         KASSERT(rw_wowner(rw) != curthread,
  412             ("rw_rlock: wlock already held for %s @ %s:%d",
  413             rw->lock_object.lo_name, file, line));
  414         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
  415 
  416 #ifdef KDTRACE_HOOKS
  417         all_time -= lockstat_nsecs(&rw->lock_object);
  418         state = rw->rw_lock;
  419 #endif
  420         for (;;) {
  421                 /*
  422                  * Handle the easy case.  If no other thread has a write
  423                  * lock, then try to bump up the count of read locks.  Note
  424                  * that we have to preserve the current state of the
  425                  * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
  426                  * read lock, then rw_lock must have changed, so restart
  427                  * the loop.  Note that this handles the case of a
  428                  * completely unlocked rwlock since such a lock is encoded
  429                  * as a read lock with no waiters.
  430                  */
  431                 v = rw->rw_lock;
  432                 if (RW_CAN_READ(v)) {
  433                         /*
  434                          * The RW_LOCK_READ_WAITERS flag should only be set
  435                          * if the lock has been unlocked and write waiters
  436                          * were present.
  437                          */
  438                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
  439                             v + RW_ONE_READER)) {
  440                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  441                                         CTR4(KTR_LOCK,
  442                                             "%s: %p succeed %p -> %p", __func__,
  443                                             rw, (void *)v,
  444                                             (void *)(v + RW_ONE_READER));
  445                                 break;
  446                         }
  447                         continue;
  448                 }
  449 #ifdef KDTRACE_HOOKS
  450                 lda.spin_cnt++;
  451 #endif
  452 #ifdef HWPMC_HOOKS
  453                 PMC_SOFT_CALL( , , lock, failed);
  454 #endif
  455                 lock_profile_obtain_lock_failed(&rw->lock_object,
  456                     &contested, &waittime);
  457 
  458 #ifdef ADAPTIVE_RWLOCKS
  459                 /*
  460                  * If the owner is running on another CPU, spin until
  461                  * the owner stops running or the state of the lock
  462                  * changes.
  463                  */
  464                 if ((v & RW_LOCK_READ) == 0) {
  465                         owner = (struct thread *)RW_OWNER(v);
  466                         if (TD_IS_RUNNING(owner)) {
  467                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  468                                         CTR3(KTR_LOCK,
  469                                             "%s: spinning on %p held by %p",
  470                                             __func__, rw, owner);
  471                                 KTR_STATE1(KTR_SCHED, "thread",
  472                                     sched_tdname(curthread), "spinning",
  473                                     "lockname:\"%s\"", rw->lock_object.lo_name);
  474                                 while ((struct thread*)RW_OWNER(rw->rw_lock) ==
  475                                     owner && TD_IS_RUNNING(owner))
  476                                         lock_delay(&lda);
  477                                 KTR_STATE0(KTR_SCHED, "thread",
  478                                     sched_tdname(curthread), "running");
  479                                 continue;
  480                         }
  481                 } else if (spintries < rowner_retries) {
  482                         spintries++;
  483                         KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
  484                             "spinning", "lockname:\"%s\"",
  485                             rw->lock_object.lo_name);
  486                         for (i = 0; i < rowner_loops; i++) {
  487                                 v = rw->rw_lock;
  488                                 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v))
  489                                         break;
  490                                 cpu_spinwait();
  491                         }
  492 #ifdef KDTRACE_HOOKS
  493                         lda.spin_cnt += rowner_loops - i;
  494 #endif
  495                         KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
  496                             "running");
  497                         if (i != rowner_loops)
  498                                 continue;
  499                 }
  500 #endif
  501 
  502                 /*
  503                  * Okay, now it's the hard case.  Some other thread already
  504                  * has a write lock or there are write waiters present,
  505                  * acquire the turnstile lock so we can begin the process
  506                  * of blocking.
  507                  */
  508                 ts = turnstile_trywait(&rw->lock_object);
  509 
  510                 /*
  511                  * The lock might have been released while we spun, so
  512                  * recheck its state and restart the loop if needed.
  513                  */
  514                 v = rw->rw_lock;
  515                 if (RW_CAN_READ(v)) {
  516                         turnstile_cancel(ts);
  517                         continue;
  518                 }
  519 
  520 #ifdef ADAPTIVE_RWLOCKS
  521                 /*
  522                  * The current lock owner might have started executing
  523                  * on another CPU (or the lock could have changed
  524                  * owners) while we were waiting on the turnstile
  525                  * chain lock.  If so, drop the turnstile lock and try
  526                  * again.
  527                  */
  528                 if ((v & RW_LOCK_READ) == 0) {
  529                         owner = (struct thread *)RW_OWNER(v);
  530                         if (TD_IS_RUNNING(owner)) {
  531                                 turnstile_cancel(ts);
  532                                 continue;
  533                         }
  534                 }
  535 #endif
  536 
  537                 /*
  538                  * The lock is held in write mode or it already has waiters.
  539                  */
  540                 MPASS(!RW_CAN_READ(v));
  541 
  542                 /*
  543                  * If the RW_LOCK_READ_WAITERS flag is already set, then
  544                  * we can go ahead and block.  If it is not set then try
  545                  * to set it.  If we fail to set it drop the turnstile
  546                  * lock and restart the loop.
  547                  */
  548                 if (!(v & RW_LOCK_READ_WAITERS)) {
  549                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  550                             v | RW_LOCK_READ_WAITERS)) {
  551                                 turnstile_cancel(ts);
  552                                 continue;
  553                         }
  554                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  555                                 CTR2(KTR_LOCK, "%s: %p set read waiters flag",
  556                                     __func__, rw);
  557                 }
  558 
  559                 /*
  560                  * We were unable to acquire the lock and the read waiters
  561                  * flag is set, so we must block on the turnstile.
  562                  */
  563                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  564                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
  565                             rw);
  566 #ifdef KDTRACE_HOOKS
  567                 sleep_time -= lockstat_nsecs(&rw->lock_object);
  568 #endif
  569                 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
  570 #ifdef KDTRACE_HOOKS
  571                 sleep_time += lockstat_nsecs(&rw->lock_object);
  572                 sleep_cnt++;
  573 #endif
  574                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  575                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
  576                             __func__, rw);
  577         }
  578 #ifdef KDTRACE_HOOKS
  579         all_time += lockstat_nsecs(&rw->lock_object);
  580         if (sleep_time)
  581                 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
  582                     LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
  583                     (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
  584 
  585         /* Record only the loops spinning and not sleeping. */
  586         if (lda.spin_cnt > sleep_cnt)
  587                 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
  588                     LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
  589                     (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
  590 #endif
  591         /*
  592          * TODO: acquire "owner of record" here.  Here be turnstile dragons
  593          * however.  turnstiles don't like owners changing between calls to
  594          * turnstile_wait() currently.
  595          */
  596         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
  597             waittime, file, line, LOCKSTAT_READER);
  598         LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
  599         WITNESS_LOCK(&rw->lock_object, 0, file, line);
  600         TD_LOCKS_INC(curthread);
  601         curthread->td_rw_rlocks++;
  602 }
  603 
  604 int
  605 __rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
  606 {
  607         struct rwlock *rw;
  608         uintptr_t x;
  609 
  610         if (SCHEDULER_STOPPED())
  611                 return (1);
  612 
  613         rw = rwlock2rw(c);
  614 
  615         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
  616             ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d",
  617             curthread, rw->lock_object.lo_name, file, line));
  618 
  619         for (;;) {
  620                 x = rw->rw_lock;
  621                 KASSERT(rw->rw_lock != RW_DESTROYED,
  622                     ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
  623                 if (!(x & RW_LOCK_READ))
  624                         break;
  625                 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
  626                         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
  627                             line);
  628                         WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
  629                         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
  630                             rw, 0, 0, file, line, LOCKSTAT_READER);
  631                         TD_LOCKS_INC(curthread);
  632                         curthread->td_rw_rlocks++;
  633                         return (1);
  634                 }
  635         }
  636 
  637         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
  638         return (0);
  639 }
  640 
  641 void
  642 _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
  643 {
  644         struct rwlock *rw;
  645         struct turnstile *ts;
  646         uintptr_t x, v, queue;
  647 
  648         if (SCHEDULER_STOPPED())
  649                 return;
  650 
  651         rw = rwlock2rw(c);
  652 
  653         KASSERT(rw->rw_lock != RW_DESTROYED,
  654             ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
  655         __rw_assert(c, RA_RLOCKED, file, line);
  656         WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
  657         LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
  658 
  659         /* TODO: drop "owner of record" here. */
  660 
  661         for (;;) {
  662                 /*
  663                  * See if there is more than one read lock held.  If so,
  664                  * just drop one and return.
  665                  */
  666                 x = rw->rw_lock;
  667                 if (RW_READERS(x) > 1) {
  668                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
  669                             x - RW_ONE_READER)) {
  670                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  671                                         CTR4(KTR_LOCK,
  672                                             "%s: %p succeeded %p -> %p",
  673                                             __func__, rw, (void *)x,
  674                                             (void *)(x - RW_ONE_READER));
  675                                 break;
  676                         }
  677                         continue;
  678                 }
  679                 /*
  680                  * If there aren't any waiters for a write lock, then try
  681                  * to drop it quickly.
  682                  */
  683                 if (!(x & RW_LOCK_WAITERS)) {
  684                         MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
  685                             RW_READERS_LOCK(1));
  686                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
  687                             RW_UNLOCKED)) {
  688                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  689                                         CTR2(KTR_LOCK, "%s: %p last succeeded",
  690                                             __func__, rw);
  691                                 break;
  692                         }
  693                         continue;
  694                 }
  695                 /*
  696                  * Ok, we know we have waiters and we think we are the
  697                  * last reader, so grab the turnstile lock.
  698                  */
  699                 turnstile_chain_lock(&rw->lock_object);
  700                 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
  701                 MPASS(v & RW_LOCK_WAITERS);
  702 
  703                 /*
  704                  * Try to drop our lock leaving the lock in a unlocked
  705                  * state.
  706                  *
  707                  * If you wanted to do explicit lock handoff you'd have to
  708                  * do it here.  You'd also want to use turnstile_signal()
  709                  * and you'd have to handle the race where a higher
  710                  * priority thread blocks on the write lock before the
  711                  * thread you wakeup actually runs and have the new thread
  712                  * "steal" the lock.  For now it's a lot simpler to just
  713                  * wakeup all of the waiters.
  714                  *
  715                  * As above, if we fail, then another thread might have
  716                  * acquired a read lock, so drop the turnstile lock and
  717                  * restart.
  718                  */
  719                 x = RW_UNLOCKED;
  720                 if (v & RW_LOCK_WRITE_WAITERS) {
  721                         queue = TS_EXCLUSIVE_QUEUE;
  722                         x |= (v & RW_LOCK_READ_WAITERS);
  723                 } else
  724                         queue = TS_SHARED_QUEUE;
  725                 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
  726                     x)) {
  727                         turnstile_chain_unlock(&rw->lock_object);
  728                         continue;
  729                 }
  730                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  731                         CTR2(KTR_LOCK, "%s: %p last succeeded with waiters",
  732                             __func__, rw);
  733 
  734                 /*
  735                  * Ok.  The lock is released and all that's left is to
  736                  * wake up the waiters.  Note that the lock might not be
  737                  * free anymore, but in that case the writers will just
  738                  * block again if they run before the new lock holder(s)
  739                  * release the lock.
  740                  */
  741                 ts = turnstile_lookup(&rw->lock_object);
  742                 MPASS(ts != NULL);
  743                 turnstile_broadcast(ts, queue);
  744                 turnstile_unpend(ts, TS_SHARED_LOCK);
  745                 turnstile_chain_unlock(&rw->lock_object);
  746                 break;
  747         }
  748         LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER);
  749         TD_LOCKS_DEC(curthread);
  750         curthread->td_rw_rlocks--;
  751 }
  752 
  753 /*
  754  * This function is called when we are unable to obtain a write lock on the
  755  * first try.  This means that at least one other thread holds either a
  756  * read or write lock.
  757  */
  758 void
  759 __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
  760     int line)
  761 {
  762         struct rwlock *rw;
  763         struct turnstile *ts;
  764 #ifdef ADAPTIVE_RWLOCKS
  765         volatile struct thread *owner;
  766         int spintries = 0;
  767         int i;
  768 #endif
  769         uintptr_t v, x;
  770 #ifdef LOCK_PROFILING
  771         uint64_t waittime = 0;
  772         int contested = 0;
  773 #endif
  774 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
  775         struct lock_delay_arg lda;
  776 #endif
  777 #ifdef KDTRACE_HOOKS
  778         uintptr_t state;
  779         u_int sleep_cnt = 0;
  780         int64_t sleep_time = 0;
  781         int64_t all_time = 0;
  782 #endif
  783 
  784         if (SCHEDULER_STOPPED())
  785                 return;
  786 
  787 #if defined(ADAPTIVE_RWLOCKS)
  788         lock_delay_arg_init(&lda, &rw_delay);
  789 #elif defined(KDTRACE_HOOKS)
  790         lock_delay_arg_init(&lda, NULL);
  791 #endif
  792         rw = rwlock2rw(c);
  793 
  794         if (rw_wlocked(rw)) {
  795                 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
  796                     ("%s: recursing but non-recursive rw %s @ %s:%d\n",
  797                     __func__, rw->lock_object.lo_name, file, line));
  798                 rw->rw_recurse++;
  799                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  800                         CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
  801                 return;
  802         }
  803 
  804         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  805                 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
  806                     rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
  807 
  808 #ifdef KDTRACE_HOOKS
  809         all_time -= lockstat_nsecs(&rw->lock_object);
  810         state = rw->rw_lock;
  811 #endif
  812         for (;;) {
  813                 if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid))
  814                         break;
  815 #ifdef KDTRACE_HOOKS
  816                 lda.spin_cnt++;
  817 #endif
  818 #ifdef HWPMC_HOOKS
  819                 PMC_SOFT_CALL( , , lock, failed);
  820 #endif
  821                 lock_profile_obtain_lock_failed(&rw->lock_object,
  822                     &contested, &waittime);
  823 #ifdef ADAPTIVE_RWLOCKS
  824                 /*
  825                  * If the lock is write locked and the owner is
  826                  * running on another CPU, spin until the owner stops
  827                  * running or the state of the lock changes.
  828                  */
  829                 v = rw->rw_lock;
  830                 owner = (struct thread *)RW_OWNER(v);
  831                 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
  832                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  833                                 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
  834                                     __func__, rw, owner);
  835                         KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
  836                             "spinning", "lockname:\"%s\"",
  837                             rw->lock_object.lo_name);
  838                         while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
  839                             TD_IS_RUNNING(owner))
  840                                 lock_delay(&lda);
  841                         KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
  842                             "running");
  843                         continue;
  844                 }
  845                 if ((v & RW_LOCK_READ) && RW_READERS(v) &&
  846                     spintries < rowner_retries) {
  847                         if (!(v & RW_LOCK_WRITE_SPINNER)) {
  848                                 if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  849                                     v | RW_LOCK_WRITE_SPINNER)) {
  850                                         continue;
  851                                 }
  852                         }
  853                         spintries++;
  854                         KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
  855                             "spinning", "lockname:\"%s\"",
  856                             rw->lock_object.lo_name);
  857                         for (i = 0; i < rowner_loops; i++) {
  858                                 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0)
  859                                         break;
  860                                 cpu_spinwait();
  861                         }
  862                         KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
  863                             "running");
  864 #ifdef KDTRACE_HOOKS
  865                         lda.spin_cnt += rowner_loops - i;
  866 #endif
  867                         if (i != rowner_loops)
  868                                 continue;
  869                 }
  870 #endif
  871                 ts = turnstile_trywait(&rw->lock_object);
  872                 v = rw->rw_lock;
  873 
  874 #ifdef ADAPTIVE_RWLOCKS
  875                 /*
  876                  * The current lock owner might have started executing
  877                  * on another CPU (or the lock could have changed
  878                  * owners) while we were waiting on the turnstile
  879                  * chain lock.  If so, drop the turnstile lock and try
  880                  * again.
  881                  */
  882                 if (!(v & RW_LOCK_READ)) {
  883                         owner = (struct thread *)RW_OWNER(v);
  884                         if (TD_IS_RUNNING(owner)) {
  885                                 turnstile_cancel(ts);
  886                                 continue;
  887                         }
  888                 }
  889 #endif
  890                 /*
  891                  * Check for the waiters flags about this rwlock.
  892                  * If the lock was released, without maintain any pending
  893                  * waiters queue, simply try to acquire it.
  894                  * If a pending waiters queue is present, claim the lock
  895                  * ownership and maintain the pending queue.
  896                  */
  897                 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
  898                 if ((v & ~x) == RW_UNLOCKED) {
  899                         x &= ~RW_LOCK_WRITE_SPINNER;
  900                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) {
  901                                 if (x)
  902                                         turnstile_claim(ts);
  903                                 else
  904                                         turnstile_cancel(ts);
  905                                 break;
  906                         }
  907                         turnstile_cancel(ts);
  908                         continue;
  909                 }
  910                 /*
  911                  * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to
  912                  * set it.  If we fail to set it, then loop back and try
  913                  * again.
  914                  */
  915                 if (!(v & RW_LOCK_WRITE_WAITERS)) {
  916                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  917                             v | RW_LOCK_WRITE_WAITERS)) {
  918                                 turnstile_cancel(ts);
  919                                 continue;
  920                         }
  921                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  922                                 CTR2(KTR_LOCK, "%s: %p set write waiters flag",
  923                                     __func__, rw);
  924                 }
  925                 /*
  926                  * We were unable to acquire the lock and the write waiters
  927                  * flag is set, so we must block on the turnstile.
  928                  */
  929                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  930                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
  931                             rw);
  932 #ifdef KDTRACE_HOOKS
  933                 sleep_time -= lockstat_nsecs(&rw->lock_object);
  934 #endif
  935                 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
  936 #ifdef KDTRACE_HOOKS
  937                 sleep_time += lockstat_nsecs(&rw->lock_object);
  938                 sleep_cnt++;
  939 #endif
  940                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  941                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
  942                             __func__, rw);
  943 #ifdef ADAPTIVE_RWLOCKS
  944                 spintries = 0;
  945 #endif
  946         }
  947 #ifdef KDTRACE_HOOKS
  948         all_time += lockstat_nsecs(&rw->lock_object);
  949         if (sleep_time)
  950                 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
  951                     LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0,
  952                     (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
  953 
  954         /* Record only the loops spinning and not sleeping. */
  955         if (lda.spin_cnt > sleep_cnt)
  956                 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
  957                     LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0,
  958                     (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
  959 #endif
  960         LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
  961             waittime, file, line, LOCKSTAT_WRITER);
  962 }
  963 
  964 /*
  965  * This function is called if the first try at releasing a write lock failed.
  966  * This means that one of the 2 waiter bits must be set indicating that at
  967  * least one thread is waiting on this lock.
  968  */
  969 void
  970 __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
  971     int line)
  972 {
  973         struct rwlock *rw;
  974         struct turnstile *ts;
  975         uintptr_t v;
  976         int queue;
  977 
  978         if (SCHEDULER_STOPPED())
  979                 return;
  980 
  981         rw = rwlock2rw(c);
  982 
  983         if (rw_wlocked(rw) && rw_recursed(rw)) {
  984                 rw->rw_recurse--;
  985                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  986                         CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
  987                 return;
  988         }
  989 
  990         KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
  991             ("%s: neither of the waiter flags are set", __func__));
  992 
  993         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  994                 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
  995 
  996         turnstile_chain_lock(&rw->lock_object);
  997         ts = turnstile_lookup(&rw->lock_object);
  998         MPASS(ts != NULL);
  999 
 1000         /*
 1001          * Use the same algo as sx locks for now.  Prefer waking up shared
 1002          * waiters if we have any over writers.  This is probably not ideal.
 1003          *
 1004          * 'v' is the value we are going to write back to rw_lock.  If we
 1005          * have waiters on both queues, we need to preserve the state of
 1006          * the waiter flag for the queue we don't wake up.  For now this is
 1007          * hardcoded for the algorithm mentioned above.
 1008          *
 1009          * In the case of both readers and writers waiting we wakeup the
 1010          * readers but leave the RW_LOCK_WRITE_WAITERS flag set.  If a
 1011          * new writer comes in before a reader it will claim the lock up
 1012          * above.  There is probably a potential priority inversion in
 1013          * there that could be worked around either by waking both queues
 1014          * of waiters or doing some complicated lock handoff gymnastics.
 1015          */
 1016         v = RW_UNLOCKED;
 1017         if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) {
 1018                 queue = TS_EXCLUSIVE_QUEUE;
 1019                 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS);
 1020         } else
 1021                 queue = TS_SHARED_QUEUE;
 1022 
 1023         /* Wake up all waiters for the specific queue. */
 1024         if (LOCK_LOG_TEST(&rw->lock_object, 0))
 1025                 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
 1026                     queue == TS_SHARED_QUEUE ? "read" : "write");
 1027         turnstile_broadcast(ts, queue);
 1028         atomic_store_rel_ptr(&rw->rw_lock, v);
 1029         turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
 1030         turnstile_chain_unlock(&rw->lock_object);
 1031 }
 1032 
 1033 /*
 1034  * Attempt to do a non-blocking upgrade from a read lock to a write
 1035  * lock.  This will only succeed if this thread holds a single read
 1036  * lock.  Returns true if the upgrade succeeded and false otherwise.
 1037  */
 1038 int
 1039 __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
 1040 {
 1041         struct rwlock *rw;
 1042         uintptr_t v, x, tid;
 1043         struct turnstile *ts;
 1044         int success;
 1045 
 1046         if (SCHEDULER_STOPPED())
 1047                 return (1);
 1048 
 1049         rw = rwlock2rw(c);
 1050 
 1051         KASSERT(rw->rw_lock != RW_DESTROYED,
 1052             ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
 1053         __rw_assert(c, RA_RLOCKED, file, line);
 1054 
 1055         /*
 1056          * Attempt to switch from one reader to a writer.  If there
 1057          * are any write waiters, then we will have to lock the
 1058          * turnstile first to prevent races with another writer
 1059          * calling turnstile_wait() before we have claimed this
 1060          * turnstile.  So, do the simple case of no waiters first.
 1061          */
 1062         tid = (uintptr_t)curthread;
 1063         success = 0;
 1064         for (;;) {
 1065                 v = rw->rw_lock;
 1066                 if (RW_READERS(v) > 1)
 1067                         break;
 1068                 if (!(v & RW_LOCK_WAITERS)) {
 1069                         success = atomic_cmpset_ptr(&rw->rw_lock, v, tid);
 1070                         if (!success)
 1071                                 continue;
 1072                         break;
 1073                 }
 1074 
 1075                 /*
 1076                  * Ok, we think we have waiters, so lock the turnstile.
 1077                  */
 1078                 ts = turnstile_trywait(&rw->lock_object);
 1079                 v = rw->rw_lock;
 1080                 if (RW_READERS(v) > 1) {
 1081                         turnstile_cancel(ts);
 1082                         break;
 1083                 }
 1084                 /*
 1085                  * Try to switch from one reader to a writer again.  This time
 1086                  * we honor the current state of the waiters flags.
 1087                  * If we obtain the lock with the flags set, then claim
 1088                  * ownership of the turnstile.
 1089                  */
 1090                 x = rw->rw_lock & RW_LOCK_WAITERS;
 1091                 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x);
 1092                 if (success) {
 1093                         if (x)
 1094                                 turnstile_claim(ts);
 1095                         else
 1096                                 turnstile_cancel(ts);
 1097                         break;
 1098                 }
 1099                 turnstile_cancel(ts);
 1100         }
 1101         LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
 1102         if (success) {
 1103                 curthread->td_rw_rlocks--;
 1104                 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 1105                     file, line);
 1106                 LOCKSTAT_RECORD0(rw__upgrade, rw);
 1107         }
 1108         return (success);
 1109 }
 1110 
 1111 /*
 1112  * Downgrade a write lock into a single read lock.
 1113  */
 1114 void
 1115 __rw_downgrade(volatile uintptr_t *c, const char *file, int line)
 1116 {
 1117         struct rwlock *rw;
 1118         struct turnstile *ts;
 1119         uintptr_t tid, v;
 1120         int rwait, wwait;
 1121 
 1122         if (SCHEDULER_STOPPED())
 1123                 return;
 1124 
 1125         rw = rwlock2rw(c);
 1126 
 1127         KASSERT(rw->rw_lock != RW_DESTROYED,
 1128             ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
 1129         __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line);
 1130 #ifndef INVARIANTS
 1131         if (rw_recursed(rw))
 1132                 panic("downgrade of a recursed lock");
 1133 #endif
 1134 
 1135         WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
 1136 
 1137         /*
 1138          * Convert from a writer to a single reader.  First we handle
 1139          * the easy case with no waiters.  If there are any waiters, we
 1140          * lock the turnstile and "disown" the lock.
 1141          */
 1142         tid = (uintptr_t)curthread;
 1143         if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
 1144                 goto out;
 1145 
 1146         /*
 1147          * Ok, we think we have waiters, so lock the turnstile so we can
 1148          * read the waiter flags without any races.
 1149          */
 1150         turnstile_chain_lock(&rw->lock_object);
 1151         v = rw->rw_lock & RW_LOCK_WAITERS;
 1152         rwait = v & RW_LOCK_READ_WAITERS;
 1153         wwait = v & RW_LOCK_WRITE_WAITERS;
 1154         MPASS(rwait | wwait);
 1155 
 1156         /*
 1157          * Downgrade from a write lock while preserving waiters flag
 1158          * and give up ownership of the turnstile.
 1159          */
 1160         ts = turnstile_lookup(&rw->lock_object);
 1161         MPASS(ts != NULL);
 1162         if (!wwait)
 1163                 v &= ~RW_LOCK_READ_WAITERS;
 1164         atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
 1165         /*
 1166          * Wake other readers if there are no writers pending.  Otherwise they
 1167          * won't be able to acquire the lock anyway.
 1168          */
 1169         if (rwait && !wwait) {
 1170                 turnstile_broadcast(ts, TS_SHARED_QUEUE);
 1171                 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
 1172         } else
 1173                 turnstile_disown(ts);
 1174         turnstile_chain_unlock(&rw->lock_object);
 1175 out:
 1176         curthread->td_rw_rlocks++;
 1177         LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
 1178         LOCKSTAT_RECORD0(rw__downgrade, rw);
 1179 }
 1180 
 1181 #ifdef INVARIANT_SUPPORT
 1182 #ifndef INVARIANTS
 1183 #undef __rw_assert
 1184 #endif
 1185 
 1186 /*
 1187  * In the non-WITNESS case, rw_assert() can only detect that at least
 1188  * *some* thread owns an rlock, but it cannot guarantee that *this*
 1189  * thread owns an rlock.
 1190  */
 1191 void
 1192 __rw_assert(const volatile uintptr_t *c, int what, const char *file, int line)
 1193 {
 1194         const struct rwlock *rw;
 1195 
 1196         if (panicstr != NULL)
 1197                 return;
 1198 
 1199         rw = rwlock2rw(c);
 1200 
 1201         switch (what) {
 1202         case RA_LOCKED:
 1203         case RA_LOCKED | RA_RECURSED:
 1204         case RA_LOCKED | RA_NOTRECURSED:
 1205         case RA_RLOCKED:
 1206         case RA_RLOCKED | RA_RECURSED:
 1207         case RA_RLOCKED | RA_NOTRECURSED:
 1208 #ifdef WITNESS
 1209                 witness_assert(&rw->lock_object, what, file, line);
 1210 #else
 1211                 /*
 1212                  * If some other thread has a write lock or we have one
 1213                  * and are asserting a read lock, fail.  Also, if no one
 1214                  * has a lock at all, fail.
 1215                  */
 1216                 if (rw->rw_lock == RW_UNLOCKED ||
 1217                     (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED ||
 1218                     rw_wowner(rw) != curthread)))
 1219                         panic("Lock %s not %slocked @ %s:%d\n",
 1220                             rw->lock_object.lo_name, (what & RA_RLOCKED) ?
 1221                             "read " : "", file, line);
 1222 
 1223                 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) {
 1224                         if (rw_recursed(rw)) {
 1225                                 if (what & RA_NOTRECURSED)
 1226                                         panic("Lock %s recursed @ %s:%d\n",
 1227                                             rw->lock_object.lo_name, file,
 1228                                             line);
 1229                         } else if (what & RA_RECURSED)
 1230                                 panic("Lock %s not recursed @ %s:%d\n",
 1231                                     rw->lock_object.lo_name, file, line);
 1232                 }
 1233 #endif
 1234                 break;
 1235         case RA_WLOCKED:
 1236         case RA_WLOCKED | RA_RECURSED:
 1237         case RA_WLOCKED | RA_NOTRECURSED:
 1238                 if (rw_wowner(rw) != curthread)
 1239                         panic("Lock %s not exclusively locked @ %s:%d\n",
 1240                             rw->lock_object.lo_name, file, line);
 1241                 if (rw_recursed(rw)) {
 1242                         if (what & RA_NOTRECURSED)
 1243                                 panic("Lock %s recursed @ %s:%d\n",
 1244                                     rw->lock_object.lo_name, file, line);
 1245                 } else if (what & RA_RECURSED)
 1246                         panic("Lock %s not recursed @ %s:%d\n",
 1247                             rw->lock_object.lo_name, file, line);
 1248                 break;
 1249         case RA_UNLOCKED:
 1250 #ifdef WITNESS
 1251                 witness_assert(&rw->lock_object, what, file, line);
 1252 #else
 1253                 /*
 1254                  * If we hold a write lock fail.  We can't reliably check
 1255                  * to see if we hold a read lock or not.
 1256                  */
 1257                 if (rw_wowner(rw) == curthread)
 1258                         panic("Lock %s exclusively locked @ %s:%d\n",
 1259                             rw->lock_object.lo_name, file, line);
 1260 #endif
 1261                 break;
 1262         default:
 1263                 panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
 1264                     line);
 1265         }
 1266 }
 1267 #endif /* INVARIANT_SUPPORT */
 1268 
 1269 #ifdef DDB
 1270 void
 1271 db_show_rwlock(const struct lock_object *lock)
 1272 {
 1273         const struct rwlock *rw;
 1274         struct thread *td;
 1275 
 1276         rw = (const struct rwlock *)lock;
 1277 
 1278         db_printf(" state: ");
 1279         if (rw->rw_lock == RW_UNLOCKED)
 1280                 db_printf("UNLOCKED\n");
 1281         else if (rw->rw_lock == RW_DESTROYED) {
 1282                 db_printf("DESTROYED\n");
 1283                 return;
 1284         } else if (rw->rw_lock & RW_LOCK_READ)
 1285                 db_printf("RLOCK: %ju locks\n",
 1286                     (uintmax_t)(RW_READERS(rw->rw_lock)));
 1287         else {
 1288                 td = rw_wowner(rw);
 1289                 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
 1290                     td->td_tid, td->td_proc->p_pid, td->td_name);
 1291                 if (rw_recursed(rw))
 1292                         db_printf(" recursed: %u\n", rw->rw_recurse);
 1293         }
 1294         db_printf(" waiters: ");
 1295         switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
 1296         case RW_LOCK_READ_WAITERS:
 1297                 db_printf("readers\n");
 1298                 break;
 1299         case RW_LOCK_WRITE_WAITERS:
 1300                 db_printf("writers\n");
 1301                 break;
 1302         case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
 1303                 db_printf("readers and writers\n");
 1304                 break;
 1305         default:
 1306                 db_printf("none\n");
 1307                 break;
 1308         }
 1309 }
 1310 
 1311 #endif

Cache object: 6cf5d92a173a064699e883b5a483cf43


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.