The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rwlock.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of the author nor the names of any co-contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 /*
   31  * Machine independent bits of reader/writer lock implementation.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/9.0/sys/kern/kern_rwlock.c 226255 2011-10-11 13:45:04Z attilio $");
   36 
   37 #include "opt_ddb.h"
   38 #include "opt_kdtrace.h"
   39 #include "opt_no_adaptive_rwlocks.h"
   40 
   41 #include <sys/param.h>
   42 #include <sys/ktr.h>
   43 #include <sys/kernel.h>
   44 #include <sys/lock.h>
   45 #include <sys/mutex.h>
   46 #include <sys/proc.h>
   47 #include <sys/rwlock.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/systm.h>
   50 #include <sys/turnstile.h>
   51 
   52 #include <machine/cpu.h>
   53 
   54 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
   55 #define ADAPTIVE_RWLOCKS
   56 #endif
   57 
   58 #ifdef ADAPTIVE_RWLOCKS
   59 #define ROWNER_RETRIES  10
   60 #define ROWNER_LOOPS    10000
   61 #endif
   62 
   63 #ifdef DDB
   64 #include <ddb/ddb.h>
   65 
   66 static void     db_show_rwlock(struct lock_object *lock);
   67 #endif
   68 static void     assert_rw(struct lock_object *lock, int what);
   69 static void     lock_rw(struct lock_object *lock, int how);
   70 #ifdef KDTRACE_HOOKS
   71 static int      owner_rw(struct lock_object *lock, struct thread **owner);
   72 #endif
   73 static int      unlock_rw(struct lock_object *lock);
   74 
   75 struct lock_class lock_class_rw = {
   76         .lc_name = "rw",
   77         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE,
   78         .lc_assert = assert_rw,
   79 #ifdef DDB
   80         .lc_ddb_show = db_show_rwlock,
   81 #endif
   82         .lc_lock = lock_rw,
   83         .lc_unlock = unlock_rw,
   84 #ifdef KDTRACE_HOOKS
   85         .lc_owner = owner_rw,
   86 #endif
   87 };
   88 
   89 /*
   90  * Return a pointer to the owning thread if the lock is write-locked or
   91  * NULL if the lock is unlocked or read-locked.
   92  */
   93 #define rw_wowner(rw)                                                   \
   94         ((rw)->rw_lock & RW_LOCK_READ ? NULL :                          \
   95             (struct thread *)RW_OWNER((rw)->rw_lock))
   96 
   97 /*
   98  * Returns if a write owner is recursed.  Write ownership is not assured
   99  * here and should be previously checked.
  100  */
  101 #define rw_recursed(rw)         ((rw)->rw_recurse != 0)
  102 
  103 /*
  104  * Return true if curthread helds the lock.
  105  */
  106 #define rw_wlocked(rw)          (rw_wowner((rw)) == curthread)
  107 
  108 /*
  109  * Return a pointer to the owning thread for this lock who should receive
  110  * any priority lent by threads that block on this lock.  Currently this
  111  * is identical to rw_wowner().
  112  */
  113 #define rw_owner(rw)            rw_wowner(rw)
  114 
  115 #ifndef INVARIANTS
  116 #define _rw_assert(rw, what, file, line)
  117 #endif
  118 
  119 void
  120 assert_rw(struct lock_object *lock, int what)
  121 {
  122 
  123         rw_assert((struct rwlock *)lock, what);
  124 }
  125 
  126 void
  127 lock_rw(struct lock_object *lock, int how)
  128 {
  129         struct rwlock *rw;
  130 
  131         rw = (struct rwlock *)lock;
  132         if (how)
  133                 rw_wlock(rw);
  134         else
  135                 rw_rlock(rw);
  136 }
  137 
  138 int
  139 unlock_rw(struct lock_object *lock)
  140 {
  141         struct rwlock *rw;
  142 
  143         rw = (struct rwlock *)lock;
  144         rw_assert(rw, RA_LOCKED | LA_NOTRECURSED);
  145         if (rw->rw_lock & RW_LOCK_READ) {
  146                 rw_runlock(rw);
  147                 return (0);
  148         } else {
  149                 rw_wunlock(rw);
  150                 return (1);
  151         }
  152 }
  153 
  154 #ifdef KDTRACE_HOOKS
  155 int
  156 owner_rw(struct lock_object *lock, struct thread **owner)
  157 {
  158         struct rwlock *rw = (struct rwlock *)lock;
  159         uintptr_t x = rw->rw_lock;
  160 
  161         *owner = rw_wowner(rw);
  162         return ((x & RW_LOCK_READ) != 0 ?  (RW_READERS(x) != 0) :
  163             (*owner != NULL));
  164 }
  165 #endif
  166 
  167 void
  168 rw_init_flags(struct rwlock *rw, const char *name, int opts)
  169 {
  170         int flags;
  171 
  172         MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET |
  173             RW_RECURSE)) == 0);
  174         ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock,
  175             ("%s: rw_lock not aligned for %s: %p", __func__, name,
  176             &rw->rw_lock));
  177 
  178         flags = LO_UPGRADABLE;
  179         if (opts & RW_DUPOK)
  180                 flags |= LO_DUPOK;
  181         if (opts & RW_NOPROFILE)
  182                 flags |= LO_NOPROFILE;
  183         if (!(opts & RW_NOWITNESS))
  184                 flags |= LO_WITNESS;
  185         if (opts & RW_RECURSE)
  186                 flags |= LO_RECURSABLE;
  187         if (opts & RW_QUIET)
  188                 flags |= LO_QUIET;
  189 
  190         rw->rw_lock = RW_UNLOCKED;
  191         rw->rw_recurse = 0;
  192         lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
  193 }
  194 
  195 void
  196 rw_destroy(struct rwlock *rw)
  197 {
  198 
  199         KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw));
  200         KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw));
  201         rw->rw_lock = RW_DESTROYED;
  202         lock_destroy(&rw->lock_object);
  203 }
  204 
  205 void
  206 rw_sysinit(void *arg)
  207 {
  208         struct rw_args *args = arg;
  209 
  210         rw_init(args->ra_rw, args->ra_desc);
  211 }
  212 
  213 void
  214 rw_sysinit_flags(void *arg)
  215 {
  216         struct rw_args_flags *args = arg;
  217 
  218         rw_init_flags(args->ra_rw, args->ra_desc, args->ra_flags);
  219 }
  220 
  221 int
  222 rw_wowned(struct rwlock *rw)
  223 {
  224 
  225         return (rw_wowner(rw) == curthread);
  226 }
  227 
  228 void
  229 _rw_wlock(struct rwlock *rw, const char *file, int line)
  230 {
  231 
  232         MPASS(curthread != NULL);
  233         KASSERT(rw->rw_lock != RW_DESTROYED,
  234             ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
  235         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
  236             line, NULL);
  237         __rw_wlock(rw, curthread, file, line);
  238         LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
  239         WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
  240         curthread->td_locks++;
  241 }
  242 
  243 int
  244 _rw_try_wlock(struct rwlock *rw, const char *file, int line)
  245 {
  246         int rval;
  247 
  248         KASSERT(rw->rw_lock != RW_DESTROYED,
  249             ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
  250 
  251         if (rw_wlocked(rw) &&
  252             (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  253                 rw->rw_recurse++;
  254                 rval = 1;
  255         } else
  256                 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
  257                     (uintptr_t)curthread);
  258 
  259         LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
  260         if (rval) {
  261                 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
  262                     file, line);
  263                 curthread->td_locks++;
  264         }
  265         return (rval);
  266 }
  267 
  268 void
  269 _rw_wunlock(struct rwlock *rw, const char *file, int line)
  270 {
  271 
  272         MPASS(curthread != NULL);
  273         KASSERT(rw->rw_lock != RW_DESTROYED,
  274             ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
  275         _rw_assert(rw, RA_WLOCKED, file, line);
  276         curthread->td_locks--;
  277         WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
  278         LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
  279             line);
  280         if (!rw_recursed(rw))
  281                 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw);
  282         __rw_wunlock(rw, curthread, file, line);
  283 }
  284 /*
  285  * Determines whether a new reader can acquire a lock.  Succeeds if the
  286  * reader already owns a read lock and the lock is locked for read to
  287  * prevent deadlock from reader recursion.  Also succeeds if the lock
  288  * is unlocked and has no writer waiters or spinners.  Failing otherwise
  289  * prioritizes writers before readers.
  290  */
  291 #define RW_CAN_READ(_rw)                                                \
  292     ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) &      \
  293     (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==  \
  294     RW_LOCK_READ)
  295 
  296 void
  297 _rw_rlock(struct rwlock *rw, const char *file, int line)
  298 {
  299         struct turnstile *ts;
  300 #ifdef ADAPTIVE_RWLOCKS
  301         volatile struct thread *owner;
  302         int spintries = 0;
  303         int i;
  304 #endif
  305 #ifdef LOCK_PROFILING
  306         uint64_t waittime = 0;
  307         int contested = 0;
  308 #endif
  309         uintptr_t v;
  310 #ifdef KDTRACE_HOOKS
  311         uint64_t spin_cnt = 0;
  312         uint64_t sleep_cnt = 0;
  313         int64_t sleep_time = 0;
  314 #endif
  315 
  316         KASSERT(rw->rw_lock != RW_DESTROYED,
  317             ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
  318         KASSERT(rw_wowner(rw) != curthread,
  319             ("%s (%s): wlock already held @ %s:%d", __func__,
  320             rw->lock_object.lo_name, file, line));
  321         WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
  322 
  323         for (;;) {
  324 #ifdef KDTRACE_HOOKS
  325                 spin_cnt++;
  326 #endif
  327                 /*
  328                  * Handle the easy case.  If no other thread has a write
  329                  * lock, then try to bump up the count of read locks.  Note
  330                  * that we have to preserve the current state of the
  331                  * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
  332                  * read lock, then rw_lock must have changed, so restart
  333                  * the loop.  Note that this handles the case of a
  334                  * completely unlocked rwlock since such a lock is encoded
  335                  * as a read lock with no waiters.
  336                  */
  337                 v = rw->rw_lock;
  338                 if (RW_CAN_READ(v)) {
  339                         /*
  340                          * The RW_LOCK_READ_WAITERS flag should only be set
  341                          * if the lock has been unlocked and write waiters
  342                          * were present.
  343                          */
  344                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
  345                             v + RW_ONE_READER)) {
  346                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  347                                         CTR4(KTR_LOCK,
  348                                             "%s: %p succeed %p -> %p", __func__,
  349                                             rw, (void *)v,
  350                                             (void *)(v + RW_ONE_READER));
  351                                 break;
  352                         }
  353                         continue;
  354                 }
  355                 lock_profile_obtain_lock_failed(&rw->lock_object,
  356                     &contested, &waittime);
  357 
  358 #ifdef ADAPTIVE_RWLOCKS
  359                 /*
  360                  * If the owner is running on another CPU, spin until
  361                  * the owner stops running or the state of the lock
  362                  * changes.
  363                  */
  364                 if ((v & RW_LOCK_READ) == 0) {
  365                         owner = (struct thread *)RW_OWNER(v);
  366                         if (TD_IS_RUNNING(owner)) {
  367                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  368                                         CTR3(KTR_LOCK,
  369                                             "%s: spinning on %p held by %p",
  370                                             __func__, rw, owner);
  371                                 while ((struct thread*)RW_OWNER(rw->rw_lock) ==
  372                                     owner && TD_IS_RUNNING(owner)) {
  373                                         cpu_spinwait();
  374 #ifdef KDTRACE_HOOKS
  375                                         spin_cnt++;
  376 #endif
  377                                 }
  378                                 continue;
  379                         }
  380                 } else if (spintries < ROWNER_RETRIES) {
  381                         spintries++;
  382                         for (i = 0; i < ROWNER_LOOPS; i++) {
  383                                 v = rw->rw_lock;
  384                                 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v))
  385                                         break;
  386                                 cpu_spinwait();
  387                         }
  388                         if (i != ROWNER_LOOPS)
  389                                 continue;
  390                 }
  391 #endif
  392 
  393                 /*
  394                  * Okay, now it's the hard case.  Some other thread already
  395                  * has a write lock or there are write waiters present,
  396                  * acquire the turnstile lock so we can begin the process
  397                  * of blocking.
  398                  */
  399                 ts = turnstile_trywait(&rw->lock_object);
  400 
  401                 /*
  402                  * The lock might have been released while we spun, so
  403                  * recheck its state and restart the loop if needed.
  404                  */
  405                 v = rw->rw_lock;
  406                 if (RW_CAN_READ(v)) {
  407                         turnstile_cancel(ts);
  408                         continue;
  409                 }
  410 
  411 #ifdef ADAPTIVE_RWLOCKS
  412                 /*
  413                  * The current lock owner might have started executing
  414                  * on another CPU (or the lock could have changed
  415                  * owners) while we were waiting on the turnstile
  416                  * chain lock.  If so, drop the turnstile lock and try
  417                  * again.
  418                  */
  419                 if ((v & RW_LOCK_READ) == 0) {
  420                         owner = (struct thread *)RW_OWNER(v);
  421                         if (TD_IS_RUNNING(owner)) {
  422                                 turnstile_cancel(ts);
  423                                 continue;
  424                         }
  425                 }
  426 #endif
  427 
  428                 /*
  429                  * The lock is held in write mode or it already has waiters.
  430                  */
  431                 MPASS(!RW_CAN_READ(v));
  432 
  433                 /*
  434                  * If the RW_LOCK_READ_WAITERS flag is already set, then
  435                  * we can go ahead and block.  If it is not set then try
  436                  * to set it.  If we fail to set it drop the turnstile
  437                  * lock and restart the loop.
  438                  */
  439                 if (!(v & RW_LOCK_READ_WAITERS)) {
  440                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  441                             v | RW_LOCK_READ_WAITERS)) {
  442                                 turnstile_cancel(ts);
  443                                 continue;
  444                         }
  445                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  446                                 CTR2(KTR_LOCK, "%s: %p set read waiters flag",
  447                                     __func__, rw);
  448                 }
  449 
  450                 /*
  451                  * We were unable to acquire the lock and the read waiters
  452                  * flag is set, so we must block on the turnstile.
  453                  */
  454                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  455                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
  456                             rw);
  457 #ifdef KDTRACE_HOOKS
  458                 sleep_time -= lockstat_nsecs();
  459 #endif
  460                 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
  461 #ifdef KDTRACE_HOOKS
  462                 sleep_time += lockstat_nsecs();
  463                 sleep_cnt++;
  464 #endif
  465                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  466                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
  467                             __func__, rw);
  468         }
  469 
  470         /*
  471          * TODO: acquire "owner of record" here.  Here be turnstile dragons
  472          * however.  turnstiles don't like owners changing between calls to
  473          * turnstile_wait() currently.
  474          */
  475         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested,
  476             waittime, file, line);
  477         LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
  478         WITNESS_LOCK(&rw->lock_object, 0, file, line);
  479         curthread->td_locks++;
  480         curthread->td_rw_rlocks++;
  481 #ifdef KDTRACE_HOOKS
  482         if (sleep_time)
  483                 LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time);
  484 
  485         /*
  486          * Record only the loops spinning and not sleeping. 
  487          */
  488         if (spin_cnt > sleep_cnt)
  489                 LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
  490 #endif
  491 }
  492 
  493 int
  494 _rw_try_rlock(struct rwlock *rw, const char *file, int line)
  495 {
  496         uintptr_t x;
  497 
  498         for (;;) {
  499                 x = rw->rw_lock;
  500                 KASSERT(rw->rw_lock != RW_DESTROYED,
  501                     ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
  502                 if (!(x & RW_LOCK_READ))
  503                         break;
  504                 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
  505                         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
  506                             line);
  507                         WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
  508                         curthread->td_locks++;
  509                         curthread->td_rw_rlocks++;
  510                         return (1);
  511                 }
  512         }
  513 
  514         LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
  515         return (0);
  516 }
  517 
  518 void
  519 _rw_runlock(struct rwlock *rw, const char *file, int line)
  520 {
  521         struct turnstile *ts;
  522         uintptr_t x, v, queue;
  523 
  524         KASSERT(rw->rw_lock != RW_DESTROYED,
  525             ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
  526         _rw_assert(rw, RA_RLOCKED, file, line);
  527         curthread->td_locks--;
  528         curthread->td_rw_rlocks--;
  529         WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
  530         LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
  531 
  532         /* TODO: drop "owner of record" here. */
  533 
  534         for (;;) {
  535                 /*
  536                  * See if there is more than one read lock held.  If so,
  537                  * just drop one and return.
  538                  */
  539                 x = rw->rw_lock;
  540                 if (RW_READERS(x) > 1) {
  541                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
  542                             x - RW_ONE_READER)) {
  543                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  544                                         CTR4(KTR_LOCK,
  545                                             "%s: %p succeeded %p -> %p",
  546                                             __func__, rw, (void *)x,
  547                                             (void *)(x - RW_ONE_READER));
  548                                 break;
  549                         }
  550                         continue;
  551                 }
  552                 /*
  553                  * If there aren't any waiters for a write lock, then try
  554                  * to drop it quickly.
  555                  */
  556                 if (!(x & RW_LOCK_WAITERS)) {
  557                         MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
  558                             RW_READERS_LOCK(1));
  559                         if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
  560                             RW_UNLOCKED)) {
  561                                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  562                                         CTR2(KTR_LOCK, "%s: %p last succeeded",
  563                                             __func__, rw);
  564                                 break;
  565                         }
  566                         continue;
  567                 }
  568                 /*
  569                  * Ok, we know we have waiters and we think we are the
  570                  * last reader, so grab the turnstile lock.
  571                  */
  572                 turnstile_chain_lock(&rw->lock_object);
  573                 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
  574                 MPASS(v & RW_LOCK_WAITERS);
  575 
  576                 /*
  577                  * Try to drop our lock leaving the lock in a unlocked
  578                  * state.
  579                  *
  580                  * If you wanted to do explicit lock handoff you'd have to
  581                  * do it here.  You'd also want to use turnstile_signal()
  582                  * and you'd have to handle the race where a higher
  583                  * priority thread blocks on the write lock before the
  584                  * thread you wakeup actually runs and have the new thread
  585                  * "steal" the lock.  For now it's a lot simpler to just
  586                  * wakeup all of the waiters.
  587                  *
  588                  * As above, if we fail, then another thread might have
  589                  * acquired a read lock, so drop the turnstile lock and
  590                  * restart.
  591                  */
  592                 x = RW_UNLOCKED;
  593                 if (v & RW_LOCK_WRITE_WAITERS) {
  594                         queue = TS_EXCLUSIVE_QUEUE;
  595                         x |= (v & RW_LOCK_READ_WAITERS);
  596                 } else
  597                         queue = TS_SHARED_QUEUE;
  598                 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
  599                     x)) {
  600                         turnstile_chain_unlock(&rw->lock_object);
  601                         continue;
  602                 }
  603                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  604                         CTR2(KTR_LOCK, "%s: %p last succeeded with waiters",
  605                             __func__, rw);
  606 
  607                 /*
  608                  * Ok.  The lock is released and all that's left is to
  609                  * wake up the waiters.  Note that the lock might not be
  610                  * free anymore, but in that case the writers will just
  611                  * block again if they run before the new lock holder(s)
  612                  * release the lock.
  613                  */
  614                 ts = turnstile_lookup(&rw->lock_object);
  615                 MPASS(ts != NULL);
  616                 turnstile_broadcast(ts, queue);
  617                 turnstile_unpend(ts, TS_SHARED_LOCK);
  618                 turnstile_chain_unlock(&rw->lock_object);
  619                 break;
  620         }
  621         LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw);
  622 }
  623 
  624 /*
  625  * This function is called when we are unable to obtain a write lock on the
  626  * first try.  This means that at least one other thread holds either a
  627  * read or write lock.
  628  */
  629 void
  630 _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
  631 {
  632         struct turnstile *ts;
  633 #ifdef ADAPTIVE_RWLOCKS
  634         volatile struct thread *owner;
  635         int spintries = 0;
  636         int i;
  637 #endif
  638         uintptr_t v, x;
  639 #ifdef LOCK_PROFILING
  640         uint64_t waittime = 0;
  641         int contested = 0;
  642 #endif
  643 #ifdef KDTRACE_HOOKS
  644         uint64_t spin_cnt = 0;
  645         uint64_t sleep_cnt = 0;
  646         int64_t sleep_time = 0;
  647 #endif
  648 
  649         if (rw_wlocked(rw)) {
  650                 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
  651                     ("%s: recursing but non-recursive rw %s @ %s:%d\n",
  652                     __func__, rw->lock_object.lo_name, file, line));
  653                 rw->rw_recurse++;
  654                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  655                         CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
  656                 return;
  657         }
  658 
  659         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  660                 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
  661                     rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
  662 
  663         while (!_rw_write_lock(rw, tid)) {
  664 #ifdef KDTRACE_HOOKS
  665                 spin_cnt++;
  666 #endif
  667                 lock_profile_obtain_lock_failed(&rw->lock_object,
  668                     &contested, &waittime);
  669 #ifdef ADAPTIVE_RWLOCKS
  670                 /*
  671                  * If the lock is write locked and the owner is
  672                  * running on another CPU, spin until the owner stops
  673                  * running or the state of the lock changes.
  674                  */
  675                 v = rw->rw_lock;
  676                 owner = (struct thread *)RW_OWNER(v);
  677                 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
  678                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  679                                 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
  680                                     __func__, rw, owner);
  681                         while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
  682                             TD_IS_RUNNING(owner)) {
  683                                 cpu_spinwait();
  684 #ifdef KDTRACE_HOOKS
  685                                 spin_cnt++;
  686 #endif
  687                         }
  688                         continue;
  689                 }
  690                 if ((v & RW_LOCK_READ) && RW_READERS(v) &&
  691                     spintries < ROWNER_RETRIES) {
  692                         if (!(v & RW_LOCK_WRITE_SPINNER)) {
  693                                 if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  694                                     v | RW_LOCK_WRITE_SPINNER)) {
  695                                         continue;
  696                                 }
  697                         }
  698                         spintries++;
  699                         for (i = 0; i < ROWNER_LOOPS; i++) {
  700                                 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0)
  701                                         break;
  702                                 cpu_spinwait();
  703                         }
  704 #ifdef KDTRACE_HOOKS
  705                         spin_cnt += ROWNER_LOOPS - i;
  706 #endif
  707                         if (i != ROWNER_LOOPS)
  708                                 continue;
  709                 }
  710 #endif
  711                 ts = turnstile_trywait(&rw->lock_object);
  712                 v = rw->rw_lock;
  713 
  714 #ifdef ADAPTIVE_RWLOCKS
  715                 /*
  716                  * The current lock owner might have started executing
  717                  * on another CPU (or the lock could have changed
  718                  * owners) while we were waiting on the turnstile
  719                  * chain lock.  If so, drop the turnstile lock and try
  720                  * again.
  721                  */
  722                 if (!(v & RW_LOCK_READ)) {
  723                         owner = (struct thread *)RW_OWNER(v);
  724                         if (TD_IS_RUNNING(owner)) {
  725                                 turnstile_cancel(ts);
  726                                 continue;
  727                         }
  728                 }
  729 #endif
  730                 /*
  731                  * Check for the waiters flags about this rwlock.
  732                  * If the lock was released, without maintain any pending
  733                  * waiters queue, simply try to acquire it.
  734                  * If a pending waiters queue is present, claim the lock
  735                  * ownership and maintain the pending queue.
  736                  */
  737                 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER);
  738                 if ((v & ~x) == RW_UNLOCKED) {
  739                         x &= ~RW_LOCK_WRITE_SPINNER;
  740                         if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) {
  741                                 if (x)
  742                                         turnstile_claim(ts);
  743                                 else
  744                                         turnstile_cancel(ts);
  745                                 break;
  746                         }
  747                         turnstile_cancel(ts);
  748                         continue;
  749                 }
  750                 /*
  751                  * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to
  752                  * set it.  If we fail to set it, then loop back and try
  753                  * again.
  754                  */
  755                 if (!(v & RW_LOCK_WRITE_WAITERS)) {
  756                         if (!atomic_cmpset_ptr(&rw->rw_lock, v,
  757                             v | RW_LOCK_WRITE_WAITERS)) {
  758                                 turnstile_cancel(ts);
  759                                 continue;
  760                         }
  761                         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  762                                 CTR2(KTR_LOCK, "%s: %p set write waiters flag",
  763                                     __func__, rw);
  764                 }
  765                 /*
  766                  * We were unable to acquire the lock and the write waiters
  767                  * flag is set, so we must block on the turnstile.
  768                  */
  769                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  770                         CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
  771                             rw);
  772 #ifdef KDTRACE_HOOKS
  773                 sleep_time -= lockstat_nsecs();
  774 #endif
  775                 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
  776 #ifdef KDTRACE_HOOKS
  777                 sleep_time += lockstat_nsecs();
  778                 sleep_cnt++;
  779 #endif
  780                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  781                         CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
  782                             __func__, rw);
  783 #ifdef ADAPTIVE_RWLOCKS
  784                 spintries = 0;
  785 #endif
  786         }
  787         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested,
  788             waittime, file, line);
  789 #ifdef KDTRACE_HOOKS
  790         if (sleep_time)
  791                 LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time);
  792 
  793         /*
  794          * Record only the loops spinning and not sleeping.
  795          */ 
  796         if (spin_cnt > sleep_cnt)
  797                 LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
  798 #endif
  799 }
  800 
  801 /*
  802  * This function is called if the first try at releasing a write lock failed.
  803  * This means that one of the 2 waiter bits must be set indicating that at
  804  * least one thread is waiting on this lock.
  805  */
  806 void
  807 _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
  808 {
  809         struct turnstile *ts;
  810         uintptr_t v;
  811         int queue;
  812 
  813         if (rw_wlocked(rw) && rw_recursed(rw)) {
  814                 rw->rw_recurse--;
  815                 if (LOCK_LOG_TEST(&rw->lock_object, 0))
  816                         CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
  817                 return;
  818         }
  819 
  820         KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
  821             ("%s: neither of the waiter flags are set", __func__));
  822 
  823         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  824                 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
  825 
  826         turnstile_chain_lock(&rw->lock_object);
  827         ts = turnstile_lookup(&rw->lock_object);
  828         MPASS(ts != NULL);
  829 
  830         /*
  831          * Use the same algo as sx locks for now.  Prefer waking up shared
  832          * waiters if we have any over writers.  This is probably not ideal.
  833          *
  834          * 'v' is the value we are going to write back to rw_lock.  If we
  835          * have waiters on both queues, we need to preserve the state of
  836          * the waiter flag for the queue we don't wake up.  For now this is
  837          * hardcoded for the algorithm mentioned above.
  838          *
  839          * In the case of both readers and writers waiting we wakeup the
  840          * readers but leave the RW_LOCK_WRITE_WAITERS flag set.  If a
  841          * new writer comes in before a reader it will claim the lock up
  842          * above.  There is probably a potential priority inversion in
  843          * there that could be worked around either by waking both queues
  844          * of waiters or doing some complicated lock handoff gymnastics.
  845          */
  846         v = RW_UNLOCKED;
  847         if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) {
  848                 queue = TS_EXCLUSIVE_QUEUE;
  849                 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS);
  850         } else
  851                 queue = TS_SHARED_QUEUE;
  852 
  853         /* Wake up all waiters for the specific queue. */
  854         if (LOCK_LOG_TEST(&rw->lock_object, 0))
  855                 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
  856                     queue == TS_SHARED_QUEUE ? "read" : "write");
  857         turnstile_broadcast(ts, queue);
  858         atomic_store_rel_ptr(&rw->rw_lock, v);
  859         turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  860         turnstile_chain_unlock(&rw->lock_object);
  861 }
  862 
  863 /*
  864  * Attempt to do a non-blocking upgrade from a read lock to a write
  865  * lock.  This will only succeed if this thread holds a single read
  866  * lock.  Returns true if the upgrade succeeded and false otherwise.
  867  */
  868 int
  869 _rw_try_upgrade(struct rwlock *rw, const char *file, int line)
  870 {
  871         uintptr_t v, x, tid;
  872         struct turnstile *ts;
  873         int success;
  874 
  875         KASSERT(rw->rw_lock != RW_DESTROYED,
  876             ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
  877         _rw_assert(rw, RA_RLOCKED, file, line);
  878 
  879         /*
  880          * Attempt to switch from one reader to a writer.  If there
  881          * are any write waiters, then we will have to lock the
  882          * turnstile first to prevent races with another writer
  883          * calling turnstile_wait() before we have claimed this
  884          * turnstile.  So, do the simple case of no waiters first.
  885          */
  886         tid = (uintptr_t)curthread;
  887         success = 0;
  888         for (;;) {
  889                 v = rw->rw_lock;
  890                 if (RW_READERS(v) > 1)
  891                         break;
  892                 if (!(v & RW_LOCK_WAITERS)) {
  893                         success = atomic_cmpset_ptr(&rw->rw_lock, v, tid);
  894                         if (!success)
  895                                 continue;
  896                         break;
  897                 }
  898 
  899                 /*
  900                  * Ok, we think we have waiters, so lock the turnstile.
  901                  */
  902                 ts = turnstile_trywait(&rw->lock_object);
  903                 v = rw->rw_lock;
  904                 if (RW_READERS(v) > 1) {
  905                         turnstile_cancel(ts);
  906                         break;
  907                 }
  908                 /*
  909                  * Try to switch from one reader to a writer again.  This time
  910                  * we honor the current state of the waiters flags.
  911                  * If we obtain the lock with the flags set, then claim
  912                  * ownership of the turnstile.
  913                  */
  914                 x = rw->rw_lock & RW_LOCK_WAITERS;
  915                 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x);
  916                 if (success) {
  917                         if (x)
  918                                 turnstile_claim(ts);
  919                         else
  920                                 turnstile_cancel(ts);
  921                         break;
  922                 }
  923                 turnstile_cancel(ts);
  924         }
  925         LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
  926         if (success) {
  927                 curthread->td_rw_rlocks--;
  928                 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
  929                     file, line);
  930                 LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw);
  931         }
  932         return (success);
  933 }
  934 
  935 /*
  936  * Downgrade a write lock into a single read lock.
  937  */
  938 void
  939 _rw_downgrade(struct rwlock *rw, const char *file, int line)
  940 {
  941         struct turnstile *ts;
  942         uintptr_t tid, v;
  943         int rwait, wwait;
  944 
  945         KASSERT(rw->rw_lock != RW_DESTROYED,
  946             ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
  947         _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
  948 #ifndef INVARIANTS
  949         if (rw_recursed(rw))
  950                 panic("downgrade of a recursed lock");
  951 #endif
  952 
  953         WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
  954 
  955         /*
  956          * Convert from a writer to a single reader.  First we handle
  957          * the easy case with no waiters.  If there are any waiters, we
  958          * lock the turnstile and "disown" the lock.
  959          */
  960         tid = (uintptr_t)curthread;
  961         if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
  962                 goto out;
  963 
  964         /*
  965          * Ok, we think we have waiters, so lock the turnstile so we can
  966          * read the waiter flags without any races.
  967          */
  968         turnstile_chain_lock(&rw->lock_object);
  969         v = rw->rw_lock & RW_LOCK_WAITERS;
  970         rwait = v & RW_LOCK_READ_WAITERS;
  971         wwait = v & RW_LOCK_WRITE_WAITERS;
  972         MPASS(rwait | wwait);
  973 
  974         /*
  975          * Downgrade from a write lock while preserving waiters flag
  976          * and give up ownership of the turnstile.
  977          */
  978         ts = turnstile_lookup(&rw->lock_object);
  979         MPASS(ts != NULL);
  980         if (!wwait)
  981                 v &= ~RW_LOCK_READ_WAITERS;
  982         atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
  983         /*
  984          * Wake other readers if there are no writers pending.  Otherwise they
  985          * won't be able to acquire the lock anyway.
  986          */
  987         if (rwait && !wwait) {
  988                 turnstile_broadcast(ts, TS_SHARED_QUEUE);
  989                 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  990         } else
  991                 turnstile_disown(ts);
  992         turnstile_chain_unlock(&rw->lock_object);
  993 out:
  994         curthread->td_rw_rlocks++;
  995         LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
  996         LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw);
  997 }
  998 
  999 #ifdef INVARIANT_SUPPORT
 1000 #ifndef INVARIANTS
 1001 #undef _rw_assert
 1002 #endif
 1003 
 1004 /*
 1005  * In the non-WITNESS case, rw_assert() can only detect that at least
 1006  * *some* thread owns an rlock, but it cannot guarantee that *this*
 1007  * thread owns an rlock.
 1008  */
 1009 void
 1010 _rw_assert(struct rwlock *rw, int what, const char *file, int line)
 1011 {
 1012 
 1013         if (panicstr != NULL)
 1014                 return;
 1015         switch (what) {
 1016         case RA_LOCKED:
 1017         case RA_LOCKED | RA_RECURSED:
 1018         case RA_LOCKED | RA_NOTRECURSED:
 1019         case RA_RLOCKED:
 1020 #ifdef WITNESS
 1021                 witness_assert(&rw->lock_object, what, file, line);
 1022 #else
 1023                 /*
 1024                  * If some other thread has a write lock or we have one
 1025                  * and are asserting a read lock, fail.  Also, if no one
 1026                  * has a lock at all, fail.
 1027                  */
 1028                 if (rw->rw_lock == RW_UNLOCKED ||
 1029                     (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED ||
 1030                     rw_wowner(rw) != curthread)))
 1031                         panic("Lock %s not %slocked @ %s:%d\n",
 1032                             rw->lock_object.lo_name, (what == RA_RLOCKED) ?
 1033                             "read " : "", file, line);
 1034 
 1035                 if (!(rw->rw_lock & RW_LOCK_READ)) {
 1036                         if (rw_recursed(rw)) {
 1037                                 if (what & RA_NOTRECURSED)
 1038                                         panic("Lock %s recursed @ %s:%d\n",
 1039                                             rw->lock_object.lo_name, file,
 1040                                             line);
 1041                         } else if (what & RA_RECURSED)
 1042                                 panic("Lock %s not recursed @ %s:%d\n",
 1043                                     rw->lock_object.lo_name, file, line);
 1044                 }
 1045 #endif
 1046                 break;
 1047         case RA_WLOCKED:
 1048         case RA_WLOCKED | RA_RECURSED:
 1049         case RA_WLOCKED | RA_NOTRECURSED:
 1050                 if (rw_wowner(rw) != curthread)
 1051                         panic("Lock %s not exclusively locked @ %s:%d\n",
 1052                             rw->lock_object.lo_name, file, line);
 1053                 if (rw_recursed(rw)) {
 1054                         if (what & RA_NOTRECURSED)
 1055                                 panic("Lock %s recursed @ %s:%d\n",
 1056                                     rw->lock_object.lo_name, file, line);
 1057                 } else if (what & RA_RECURSED)
 1058                         panic("Lock %s not recursed @ %s:%d\n",
 1059                             rw->lock_object.lo_name, file, line);
 1060                 break;
 1061         case RA_UNLOCKED:
 1062 #ifdef WITNESS
 1063                 witness_assert(&rw->lock_object, what, file, line);
 1064 #else
 1065                 /*
 1066                  * If we hold a write lock fail.  We can't reliably check
 1067                  * to see if we hold a read lock or not.
 1068                  */
 1069                 if (rw_wowner(rw) == curthread)
 1070                         panic("Lock %s exclusively locked @ %s:%d\n",
 1071                             rw->lock_object.lo_name, file, line);
 1072 #endif
 1073                 break;
 1074         default:
 1075                 panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
 1076                     line);
 1077         }
 1078 }
 1079 #endif /* INVARIANT_SUPPORT */
 1080 
 1081 #ifdef DDB
 1082 void
 1083 db_show_rwlock(struct lock_object *lock)
 1084 {
 1085         struct rwlock *rw;
 1086         struct thread *td;
 1087 
 1088         rw = (struct rwlock *)lock;
 1089 
 1090         db_printf(" state: ");
 1091         if (rw->rw_lock == RW_UNLOCKED)
 1092                 db_printf("UNLOCKED\n");
 1093         else if (rw->rw_lock == RW_DESTROYED) {
 1094                 db_printf("DESTROYED\n");
 1095                 return;
 1096         } else if (rw->rw_lock & RW_LOCK_READ)
 1097                 db_printf("RLOCK: %ju locks\n",
 1098                     (uintmax_t)(RW_READERS(rw->rw_lock)));
 1099         else {
 1100                 td = rw_wowner(rw);
 1101                 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
 1102                     td->td_tid, td->td_proc->p_pid, td->td_name);
 1103                 if (rw_recursed(rw))
 1104                         db_printf(" recursed: %u\n", rw->rw_recurse);
 1105         }
 1106         db_printf(" waiters: ");
 1107         switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
 1108         case RW_LOCK_READ_WAITERS:
 1109                 db_printf("readers\n");
 1110                 break;
 1111         case RW_LOCK_WRITE_WAITERS:
 1112                 db_printf("writers\n");
 1113                 break;
 1114         case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
 1115                 db_printf("readers and writers\n");
 1116                 break;
 1117         default:
 1118                 db_printf("none\n");
 1119                 break;
 1120         }
 1121 }
 1122 
 1123 #endif

Cache object: 85c1791db7cbc3d8ee98a9bbaf4fadc8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.