The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_lock.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice(s), this list of conditions and the following disclaimer as
   10  *    the first lines of this file unmodified other than the possible
   11  *    addition of one or more copyright notices.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice(s), this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
   17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
   20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   26  * DAMAGE.
   27  */
   28 
   29 #include "opt_adaptive_lockmgrs.h"
   30 #include "opt_ddb.h"
   31 #include "opt_kdtrace.h"
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/8.2/sys/kern/kern_lock.c 202576 2010-01-18 14:43:44Z attilio $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/ktr.h>
   38 #include <sys/linker_set.h>
   39 #include <sys/lock.h>
   40 #include <sys/lock_profile.h>
   41 #include <sys/lockmgr.h>
   42 #include <sys/mutex.h>
   43 #include <sys/proc.h>
   44 #include <sys/sleepqueue.h>
   45 #ifdef DEBUG_LOCKS
   46 #include <sys/stack.h>
   47 #endif
   48 #include <sys/sysctl.h>
   49 #include <sys/systm.h>
   50 
   51 #include <machine/cpu.h>
   52 
   53 #ifdef DDB
   54 #include <ddb/ddb.h>
   55 #endif
   56 
   57 CTASSERT(((LK_ADAPTIVE | LK_EXSLPFAIL | LK_NOSHARE) & LO_CLASSFLAGS) ==
   58     (LK_ADAPTIVE | LK_EXSLPFAIL | LK_NOSHARE));
   59 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
   60     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
   61 
   62 #define SQ_EXCLUSIVE_QUEUE      0
   63 #define SQ_SHARED_QUEUE         1
   64 
   65 #ifdef ADAPTIVE_LOCKMGRS
   66 #define ALK_RETRIES             10
   67 #define ALK_LOOPS               10000
   68 #endif
   69 
   70 #ifndef INVARIANTS
   71 #define _lockmgr_assert(lk, what, file, line)
   72 #define TD_LOCKS_INC(td)
   73 #define TD_LOCKS_DEC(td)
   74 #else
   75 #define TD_LOCKS_INC(td)        ((td)->td_locks++)
   76 #define TD_LOCKS_DEC(td)        ((td)->td_locks--)
   77 #endif
   78 #define TD_SLOCKS_INC(td)       ((td)->td_lk_slocks++)
   79 #define TD_SLOCKS_DEC(td)       ((td)->td_lk_slocks--)
   80 
   81 #ifndef DEBUG_LOCKS
   82 #define STACK_PRINT(lk)
   83 #define STACK_SAVE(lk)
   84 #define STACK_ZERO(lk)
   85 #else
   86 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
   87 #define STACK_SAVE(lk)  stack_save(&(lk)->lk_stack)
   88 #define STACK_ZERO(lk)  stack_zero(&(lk)->lk_stack)
   89 #endif
   90 
   91 #define LOCK_LOG2(lk, string, arg1, arg2)                               \
   92         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
   93                 CTR2(KTR_LOCK, (string), (arg1), (arg2))
   94 #define LOCK_LOG3(lk, string, arg1, arg2, arg3)                         \
   95         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
   96                 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
   97 
   98 #define GIANT_DECLARE                                                   \
   99         int _i = 0;                                                     \
  100         WITNESS_SAVE_DECL(Giant)
  101 #define GIANT_RESTORE() do {                                            \
  102         if (_i > 0) {                                                   \
  103                 while (_i--)                                            \
  104                         mtx_lock(&Giant);                               \
  105                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
  106         }                                                               \
  107 } while (0)
  108 #define GIANT_SAVE() do {                                               \
  109         if (mtx_owned(&Giant)) {                                        \
  110                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
  111                 while (mtx_owned(&Giant)) {                             \
  112                         _i++;                                           \
  113                         mtx_unlock(&Giant);                             \
  114                 }                                                       \
  115         }                                                               \
  116 } while (0)
  117 
  118 #define LK_CAN_SHARE(x)                                                 \
  119         (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 ||      \
  120         ((x) & LK_EXCLUSIVE_SPINNERS) == 0 ||                           \
  121         curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT)))
  122 #define LK_TRYOP(x)                                                     \
  123         ((x) & LK_NOWAIT)
  124 
  125 #define LK_CAN_WITNESS(x)                                               \
  126         (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
  127 #define LK_TRYWIT(x)                                                    \
  128         (LK_TRYOP(x) ? LOP_TRYLOCK : 0)
  129 
  130 #define LK_CAN_ADAPT(lk, f)                                             \
  131         (((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&             \
  132         ((f) & LK_SLEEPFAIL) == 0)
  133 
  134 #define lockmgr_disowned(lk)                                            \
  135         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
  136 
  137 #define lockmgr_xlocked(lk)                                             \
  138         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
  139 
  140 static void      assert_lockmgr(struct lock_object *lock, int how);
  141 #ifdef DDB
  142 static void      db_show_lockmgr(struct lock_object *lock);
  143 #endif
  144 static void      lock_lockmgr(struct lock_object *lock, int how);
  145 #ifdef KDTRACE_HOOKS
  146 static int       owner_lockmgr(struct lock_object *lock, struct thread **owner);
  147 #endif
  148 static int       unlock_lockmgr(struct lock_object *lock);
  149 
  150 struct lock_class lock_class_lockmgr = {
  151         .lc_name = "lockmgr",
  152         .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
  153         .lc_assert = assert_lockmgr,
  154 #ifdef DDB
  155         .lc_ddb_show = db_show_lockmgr,
  156 #endif
  157         .lc_lock = lock_lockmgr,
  158         .lc_unlock = unlock_lockmgr,
  159 #ifdef KDTRACE_HOOKS
  160         .lc_owner = owner_lockmgr,
  161 #endif
  162 };
  163 
  164 static __inline struct thread *
  165 lockmgr_xholder(struct lock *lk)
  166 {
  167         uintptr_t x;
  168 
  169         x = lk->lk_lock;
  170         return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
  171 }
  172 
  173 /*
  174  * It assumes sleepq_lock held and returns with this one unheld.
  175  * It also assumes the generic interlock is sane and previously checked.
  176  * If LK_INTERLOCK is specified the interlock is not reacquired after the
  177  * sleep.
  178  */
  179 static __inline int
  180 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
  181     const char *wmesg, int pri, int timo, int queue)
  182 {
  183         GIANT_DECLARE;
  184         struct lock_class *class;
  185         int catch, error;
  186 
  187         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
  188         catch = pri & PCATCH;
  189         pri &= PRIMASK;
  190         error = 0;
  191 
  192         LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
  193             (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
  194 
  195         if (flags & LK_INTERLOCK)
  196                 class->lc_unlock(ilk);
  197 
  198         /*
  199          * LK_EXSLPFAIL is not invariant during the lock pattern but it is
  200          * always protected by the sleepqueue spinlock, thus it is safe to
  201          * handle within the lo_flags.
  202          */
  203         if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
  204                 lk->lock_object.lo_flags |= LK_EXSLPFAIL;
  205         GIANT_SAVE();
  206         sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
  207             SLEEPQ_INTERRUPTIBLE : 0), queue);
  208         if ((flags & LK_TIMELOCK) && timo)
  209                 sleepq_set_timeout(&lk->lock_object, timo);
  210 
  211         /*
  212          * Decisional switch for real sleeping.
  213          */
  214         if ((flags & LK_TIMELOCK) && timo && catch)
  215                 error = sleepq_timedwait_sig(&lk->lock_object, pri);
  216         else if ((flags & LK_TIMELOCK) && timo)
  217                 error = sleepq_timedwait(&lk->lock_object, pri);
  218         else if (catch)
  219                 error = sleepq_wait_sig(&lk->lock_object, pri);
  220         else
  221                 sleepq_wait(&lk->lock_object, pri);
  222         GIANT_RESTORE();
  223         if ((flags & LK_SLEEPFAIL) && error == 0)
  224                 error = ENOLCK;
  225 
  226         return (error);
  227 }
  228 
  229 static __inline int
  230 wakeupshlk(struct lock *lk, const char *file, int line)
  231 {
  232         uintptr_t v, x;
  233         u_int realexslp;
  234         int queue, wakeup_swapper;
  235 
  236         TD_LOCKS_DEC(curthread);
  237         TD_SLOCKS_DEC(curthread);
  238         WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
  239         LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
  240 
  241         wakeup_swapper = 0;
  242         for (;;) {
  243                 x = lk->lk_lock;
  244 
  245                 /*
  246                  * If there is more than one shared lock held, just drop one
  247                  * and return.
  248                  */
  249                 if (LK_SHARERS(x) > 1) {
  250                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, x,
  251                             x - LK_ONE_SHARER))
  252                                 break;
  253                         continue;
  254                 }
  255 
  256                 /*
  257                  * If there are not waiters on the exclusive queue, drop the
  258                  * lock quickly.
  259                  */
  260                 if ((x & LK_ALL_WAITERS) == 0) {
  261                         MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
  262                             LK_SHARERS_LOCK(1));
  263                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, x, LK_UNLOCKED))
  264                                 break;
  265                         continue;
  266                 }
  267 
  268                 /*
  269                  * We should have a sharer with waiters, so enter the hard
  270                  * path in order to handle wakeups correctly.
  271                  */
  272                 sleepq_lock(&lk->lock_object);
  273                 x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
  274                 v = LK_UNLOCKED;
  275 
  276                 /*
  277                  * If the lock has exclusive waiters, give them preference in
  278                  * order to avoid deadlock with shared runners up.
  279                  * If interruptible sleeps left the exclusive queue empty
  280                  * avoid a starvation for the threads sleeping on the shared
  281                  * queue by giving them precedence and cleaning up the
  282                  * exclusive waiters bit anyway.
  283                  * Please note that the LK_EXSLPFAIL flag may be lying about
  284                  * the real presence of waiters with the LK_SLEEPFAIL flag on
  285                  * because they may be used in conjuction with interruptible
  286                  * sleeps.
  287                  */
  288                 realexslp = sleepq_sleepcnt(&lk->lock_object,
  289                     SQ_EXCLUSIVE_QUEUE);
  290                 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
  291                         if ((lk->lock_object.lo_flags & LK_EXSLPFAIL) == 0) {
  292                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
  293                                 queue = SQ_EXCLUSIVE_QUEUE;
  294                                 v |= (x & LK_SHARED_WAITERS);
  295                         } else {
  296                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
  297                                 LOCK_LOG2(lk,
  298                                     "%s: %p has only LK_SLEEPFAIL sleepers",
  299                                     __func__, lk);
  300                                 LOCK_LOG2(lk,
  301                             "%s: %p waking up threads on the exclusive queue",
  302                                     __func__, lk);
  303                                 wakeup_swapper =
  304                                     sleepq_broadcast(&lk->lock_object,
  305                                     SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
  306                                 queue = SQ_SHARED_QUEUE;
  307                         }
  308                                 
  309                 } else {
  310 
  311                         /*
  312                          * Exclusive waiters sleeping with LK_SLEEPFAIL on
  313                          * and using interruptible sleeps/timeout may have
  314                          * left spourious LK_EXSLPFAIL flag on, so clean
  315                          * it up anyway.
  316                          */
  317                         lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
  318                         queue = SQ_SHARED_QUEUE;
  319                 }
  320 
  321                 if (!atomic_cmpset_rel_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
  322                     v)) {
  323                         sleepq_release(&lk->lock_object);
  324                         continue;
  325                 }
  326                 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
  327                     __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
  328                     "exclusive");
  329                 wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
  330                     0, queue);
  331                 sleepq_release(&lk->lock_object);
  332                 break;
  333         }
  334 
  335         lock_profile_release_lock(&lk->lock_object);
  336         return (wakeup_swapper);
  337 }
  338 
  339 static void
  340 assert_lockmgr(struct lock_object *lock, int what)
  341 {
  342 
  343         panic("lockmgr locks do not support assertions");
  344 }
  345 
  346 static void
  347 lock_lockmgr(struct lock_object *lock, int how)
  348 {
  349 
  350         panic("lockmgr locks do not support sleep interlocking");
  351 }
  352 
  353 static int
  354 unlock_lockmgr(struct lock_object *lock)
  355 {
  356 
  357         panic("lockmgr locks do not support sleep interlocking");
  358 }
  359 
  360 #ifdef KDTRACE_HOOKS
  361 static int
  362 owner_lockmgr(struct lock_object *lock, struct thread **owner)
  363 {
  364 
  365         panic("lockmgr locks do not support owner inquiring");
  366 }
  367 #endif
  368 
  369 void
  370 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
  371 {
  372         int iflags;
  373 
  374         MPASS((flags & ~LK_INIT_MASK) == 0);
  375         ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
  376             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
  377             &lk->lk_lock));
  378 
  379         iflags = LO_SLEEPABLE | LO_UPGRADABLE;
  380         if (flags & LK_CANRECURSE)
  381                 iflags |= LO_RECURSABLE;
  382         if ((flags & LK_NODUP) == 0)
  383                 iflags |= LO_DUPOK;
  384         if (flags & LK_NOPROFILE)
  385                 iflags |= LO_NOPROFILE;
  386         if ((flags & LK_NOWITNESS) == 0)
  387                 iflags |= LO_WITNESS;
  388         if (flags & LK_QUIET)
  389                 iflags |= LO_QUIET;
  390         iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
  391 
  392         lk->lk_lock = LK_UNLOCKED;
  393         lk->lk_recurse = 0;
  394         lk->lk_timo = timo;
  395         lk->lk_pri = pri;
  396         lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
  397         STACK_ZERO(lk);
  398 }
  399 
  400 void
  401 lockdestroy(struct lock *lk)
  402 {
  403 
  404         KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
  405         KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
  406         KASSERT((lk->lock_object.lo_flags & LK_EXSLPFAIL) == 0,
  407             ("lockmgr still exclusive waiters"));
  408         lock_destroy(&lk->lock_object);
  409 }
  410 
  411 int
  412 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
  413     const char *wmesg, int pri, int timo, const char *file, int line)
  414 {
  415         GIANT_DECLARE;
  416         struct lock_class *class;
  417         const char *iwmesg;
  418         uintptr_t tid, v, x;
  419         u_int op, realexslp;
  420         int error, ipri, itimo, queue, wakeup_swapper;
  421 #ifdef LOCK_PROFILING
  422         uint64_t waittime = 0;
  423         int contested = 0;
  424 #endif
  425 #ifdef ADAPTIVE_LOCKMGRS
  426         volatile struct thread *owner;
  427         u_int i, spintries = 0;
  428 #endif
  429 
  430         error = 0;
  431         tid = (uintptr_t)curthread;
  432         op = (flags & LK_TYPE_MASK);
  433         iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
  434         ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
  435         itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
  436 
  437         MPASS((flags & ~LK_TOTAL_MASK) == 0);
  438         KASSERT((op & (op - 1)) == 0,
  439             ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
  440         KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
  441             (op != LK_DOWNGRADE && op != LK_RELEASE),
  442             ("%s: Invalid flags in regard of the operation desired @ %s:%d",
  443             __func__, file, line));
  444         KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
  445             ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
  446             __func__, file, line));
  447 
  448         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
  449         if (panicstr != NULL) {
  450                 if (flags & LK_INTERLOCK)
  451                         class->lc_unlock(ilk);
  452                 return (0);
  453         }
  454 
  455         if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
  456                 op = LK_EXCLUSIVE;
  457 
  458         wakeup_swapper = 0;
  459         switch (op) {
  460         case LK_SHARED:
  461                 if (LK_CAN_WITNESS(flags))
  462                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
  463                             file, line, ilk);
  464                 for (;;) {
  465                         x = lk->lk_lock;
  466 
  467                         /*
  468                          * If no other thread has an exclusive lock, or
  469                          * no exclusive waiter is present, bump the count of
  470                          * sharers.  Since we have to preserve the state of
  471                          * waiters, if we fail to acquire the shared lock
  472                          * loop back and retry.
  473                          */
  474                         if (LK_CAN_SHARE(x)) {
  475                                 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
  476                                     x + LK_ONE_SHARER))
  477                                         break;
  478                                 continue;
  479                         }
  480                         lock_profile_obtain_lock_failed(&lk->lock_object,
  481                             &contested, &waittime);
  482 
  483                         /*
  484                          * If the lock is already held by curthread in
  485                          * exclusive way avoid a deadlock.
  486                          */
  487                         if (LK_HOLDER(x) == tid) {
  488                                 LOCK_LOG2(lk,
  489                                     "%s: %p already held in exclusive mode",
  490                                     __func__, lk);
  491                                 error = EDEADLK;
  492                                 break;
  493                         }
  494 
  495                         /*
  496                          * If the lock is expected to not sleep just give up
  497                          * and return.
  498                          */
  499                         if (LK_TRYOP(flags)) {
  500                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
  501                                     __func__, lk);
  502                                 error = EBUSY;
  503                                 break;
  504                         }
  505 
  506 #ifdef ADAPTIVE_LOCKMGRS
  507                         /*
  508                          * If the owner is running on another CPU, spin until
  509                          * the owner stops running or the state of the lock
  510                          * changes.  We need a double-state handle here
  511                          * because for a failed acquisition the lock can be
  512                          * either held in exclusive mode or shared mode
  513                          * (for the writer starvation avoidance technique).
  514                          */
  515                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
  516                             LK_HOLDER(x) != LK_KERNPROC) {
  517                                 owner = (struct thread *)LK_HOLDER(x);
  518                                 if (LOCK_LOG_TEST(&lk->lock_object, 0))
  519                                         CTR3(KTR_LOCK,
  520                                             "%s: spinning on %p held by %p",
  521                                             __func__, lk, owner);
  522 
  523                                 /*
  524                                  * If we are holding also an interlock drop it
  525                                  * in order to avoid a deadlock if the lockmgr
  526                                  * owner is adaptively spinning on the
  527                                  * interlock itself.
  528                                  */
  529                                 if (flags & LK_INTERLOCK) {
  530                                         class->lc_unlock(ilk);
  531                                         flags &= ~LK_INTERLOCK;
  532                                 }
  533                                 GIANT_SAVE();
  534                                 while (LK_HOLDER(lk->lk_lock) ==
  535                                     (uintptr_t)owner && TD_IS_RUNNING(owner))
  536                                         cpu_spinwait();
  537                                 GIANT_RESTORE();
  538                                 continue;
  539                         } else if (LK_CAN_ADAPT(lk, flags) &&
  540                             (x & LK_SHARE) != 0 && LK_SHARERS(x) &&
  541                             spintries < ALK_RETRIES) {
  542                                 if (flags & LK_INTERLOCK) {
  543                                         class->lc_unlock(ilk);
  544                                         flags &= ~LK_INTERLOCK;
  545                                 }
  546                                 GIANT_SAVE();
  547                                 spintries++;
  548                                 for (i = 0; i < ALK_LOOPS; i++) {
  549                                         if (LOCK_LOG_TEST(&lk->lock_object, 0))
  550                                                 CTR4(KTR_LOCK,
  551                                     "%s: shared spinning on %p with %u and %u",
  552                                                     __func__, lk, spintries, i);
  553                                         x = lk->lk_lock;
  554                                         if ((x & LK_SHARE) == 0 ||
  555                                             LK_CAN_SHARE(x) != 0)
  556                                                 break;
  557                                         cpu_spinwait();
  558                                 }
  559                                 GIANT_RESTORE();
  560                                 if (i != ALK_LOOPS)
  561                                         continue;
  562                         }
  563 #endif
  564 
  565                         /*
  566                          * Acquire the sleepqueue chain lock because we
  567                          * probabilly will need to manipulate waiters flags.
  568                          */
  569                         sleepq_lock(&lk->lock_object);
  570                         x = lk->lk_lock;
  571 
  572                         /*
  573                          * if the lock can be acquired in shared mode, try
  574                          * again.
  575                          */
  576                         if (LK_CAN_SHARE(x)) {
  577                                 sleepq_release(&lk->lock_object);
  578                                 continue;
  579                         }
  580 
  581 #ifdef ADAPTIVE_LOCKMGRS
  582                         /*
  583                          * The current lock owner might have started executing
  584                          * on another CPU (or the lock could have changed
  585                          * owner) while we were waiting on the turnstile
  586                          * chain lock.  If so, drop the turnstile lock and try
  587                          * again.
  588                          */
  589                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
  590                             LK_HOLDER(x) != LK_KERNPROC) {
  591                                 owner = (struct thread *)LK_HOLDER(x);
  592                                 if (TD_IS_RUNNING(owner)) {
  593                                         sleepq_release(&lk->lock_object);
  594                                         continue;
  595                                 }
  596                         }
  597 #endif
  598 
  599                         /*
  600                          * Try to set the LK_SHARED_WAITERS flag.  If we fail,
  601                          * loop back and retry.
  602                          */
  603                         if ((x & LK_SHARED_WAITERS) == 0) {
  604                                 if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
  605                                     x | LK_SHARED_WAITERS)) {
  606                                         sleepq_release(&lk->lock_object);
  607                                         continue;
  608                                 }
  609                                 LOCK_LOG2(lk, "%s: %p set shared waiters flag",
  610                                     __func__, lk);
  611                         }
  612 
  613                         /*
  614                          * As far as we have been unable to acquire the
  615                          * shared lock and the shared waiters flag is set,
  616                          * we will sleep.
  617                          */
  618                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
  619                             SQ_SHARED_QUEUE);
  620                         flags &= ~LK_INTERLOCK;
  621                         if (error) {
  622                                 LOCK_LOG3(lk,
  623                                     "%s: interrupted sleep for %p with %d",
  624                                     __func__, lk, error);
  625                                 break;
  626                         }
  627                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
  628                             __func__, lk);
  629                 }
  630                 if (error == 0) {
  631                         lock_profile_obtain_lock_success(&lk->lock_object,
  632                             contested, waittime, file, line);
  633                         LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
  634                             line);
  635                         WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file,
  636                             line);
  637                         TD_LOCKS_INC(curthread);
  638                         TD_SLOCKS_INC(curthread);
  639                         STACK_SAVE(lk);
  640                 }
  641                 break;
  642         case LK_UPGRADE:
  643                 _lockmgr_assert(lk, KA_SLOCKED, file, line);
  644                 v = lk->lk_lock;
  645                 x = v & LK_ALL_WAITERS;
  646                 v &= LK_EXCLUSIVE_SPINNERS;
  647 
  648                 /*
  649                  * Try to switch from one shared lock to an exclusive one.
  650                  * We need to preserve waiters flags during the operation.
  651                  */
  652                 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
  653                     tid | x)) {
  654                         LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
  655                             line);
  656                         WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
  657                             LK_TRYWIT(flags), file, line);
  658                         TD_SLOCKS_DEC(curthread);
  659                         break;
  660                 }
  661 
  662                 /*
  663                  * We have been unable to succeed in upgrading, so just
  664                  * give up the shared lock.
  665                  */
  666                 wakeup_swapper |= wakeupshlk(lk, file, line);
  667 
  668                 /* FALLTHROUGH */
  669         case LK_EXCLUSIVE:
  670                 if (LK_CAN_WITNESS(flags))
  671                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
  672                             LOP_EXCLUSIVE, file, line, ilk);
  673 
  674                 /*
  675                  * If curthread already holds the lock and this one is
  676                  * allowed to recurse, simply recurse on it.
  677                  */
  678                 if (lockmgr_xlocked(lk)) {
  679                         if ((flags & LK_CANRECURSE) == 0 &&
  680                             (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
  681 
  682                                 /*
  683                                  * If the lock is expected to not panic just
  684                                  * give up and return.
  685                                  */
  686                                 if (LK_TRYOP(flags)) {
  687                                         LOCK_LOG2(lk,
  688                                             "%s: %p fails the try operation",
  689                                             __func__, lk);
  690                                         error = EBUSY;
  691                                         break;
  692                                 }
  693                                 if (flags & LK_INTERLOCK)
  694                                         class->lc_unlock(ilk);
  695                 panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
  696                                     __func__, iwmesg, file, line);
  697                         }
  698                         lk->lk_recurse++;
  699                         LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
  700                         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
  701                             lk->lk_recurse, file, line);
  702                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
  703                             LK_TRYWIT(flags), file, line);
  704                         TD_LOCKS_INC(curthread);
  705                         break;
  706                 }
  707 
  708                 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
  709                     tid)) {
  710                         lock_profile_obtain_lock_failed(&lk->lock_object,
  711                             &contested, &waittime);
  712 
  713                         /*
  714                          * If the lock is expected to not sleep just give up
  715                          * and return.
  716                          */
  717                         if (LK_TRYOP(flags)) {
  718                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
  719                                     __func__, lk);
  720                                 error = EBUSY;
  721                                 break;
  722                         }
  723 
  724 #ifdef ADAPTIVE_LOCKMGRS
  725                         /*
  726                          * If the owner is running on another CPU, spin until
  727                          * the owner stops running or the state of the lock
  728                          * changes.
  729                          */
  730                         x = lk->lk_lock;
  731                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
  732                             LK_HOLDER(x) != LK_KERNPROC) {
  733                                 owner = (struct thread *)LK_HOLDER(x);
  734                                 if (LOCK_LOG_TEST(&lk->lock_object, 0))
  735                                         CTR3(KTR_LOCK,
  736                                             "%s: spinning on %p held by %p",
  737                                             __func__, lk, owner);
  738 
  739                                 /*
  740                                  * If we are holding also an interlock drop it
  741                                  * in order to avoid a deadlock if the lockmgr
  742                                  * owner is adaptively spinning on the
  743                                  * interlock itself.
  744                                  */
  745                                 if (flags & LK_INTERLOCK) {
  746                                         class->lc_unlock(ilk);
  747                                         flags &= ~LK_INTERLOCK;
  748                                 }
  749                                 GIANT_SAVE();
  750                                 while (LK_HOLDER(lk->lk_lock) ==
  751                                     (uintptr_t)owner && TD_IS_RUNNING(owner))
  752                                         cpu_spinwait();
  753                                 GIANT_RESTORE();
  754                                 continue;
  755                         } else if (LK_CAN_ADAPT(lk, flags) &&
  756                             (x & LK_SHARE) != 0 && LK_SHARERS(x) &&
  757                             spintries < ALK_RETRIES) {
  758                                 if ((x & LK_EXCLUSIVE_SPINNERS) == 0 &&
  759                                     !atomic_cmpset_ptr(&lk->lk_lock, x,
  760                                     x | LK_EXCLUSIVE_SPINNERS))
  761                                         continue;
  762                                 if (flags & LK_INTERLOCK) {
  763                                         class->lc_unlock(ilk);
  764                                         flags &= ~LK_INTERLOCK;
  765                                 }
  766                                 GIANT_SAVE();
  767                                 spintries++;
  768                                 for (i = 0; i < ALK_LOOPS; i++) {
  769                                         if (LOCK_LOG_TEST(&lk->lock_object, 0))
  770                                                 CTR4(KTR_LOCK,
  771                                     "%s: shared spinning on %p with %u and %u",
  772                                                     __func__, lk, spintries, i);
  773                                         if ((lk->lk_lock &
  774                                             LK_EXCLUSIVE_SPINNERS) == 0)
  775                                                 break;
  776                                         cpu_spinwait();
  777                                 }
  778                                 GIANT_RESTORE();
  779                                 if (i != ALK_LOOPS)
  780                                         continue;
  781                         }
  782 #endif
  783 
  784                         /*
  785                          * Acquire the sleepqueue chain lock because we
  786                          * probabilly will need to manipulate waiters flags.
  787                          */
  788                         sleepq_lock(&lk->lock_object);
  789                         x = lk->lk_lock;
  790 
  791                         /*
  792                          * if the lock has been released while we spun on
  793                          * the sleepqueue chain lock just try again.
  794                          */
  795                         if (x == LK_UNLOCKED) {
  796                                 sleepq_release(&lk->lock_object);
  797                                 continue;
  798                         }
  799 
  800 #ifdef ADAPTIVE_LOCKMGRS
  801                         /*
  802                          * The current lock owner might have started executing
  803                          * on another CPU (or the lock could have changed
  804                          * owner) while we were waiting on the turnstile
  805                          * chain lock.  If so, drop the turnstile lock and try
  806                          * again.
  807                          */
  808                         if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
  809                             LK_HOLDER(x) != LK_KERNPROC) {
  810                                 owner = (struct thread *)LK_HOLDER(x);
  811                                 if (TD_IS_RUNNING(owner)) {
  812                                         sleepq_release(&lk->lock_object);
  813                                         continue;
  814                                 }
  815                         }
  816 #endif
  817 
  818                         /*
  819                          * The lock can be in the state where there is a
  820                          * pending queue of waiters, but still no owner.
  821                          * This happens when the lock is contested and an
  822                          * owner is going to claim the lock.
  823                          * If curthread is the one successfully acquiring it
  824                          * claim lock ownership and return, preserving waiters
  825                          * flags.
  826                          */
  827                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
  828                         if ((x & ~v) == LK_UNLOCKED) {
  829                                 v &= ~LK_EXCLUSIVE_SPINNERS;
  830                                 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
  831                                     tid | v)) {
  832                                         sleepq_release(&lk->lock_object);
  833                                         LOCK_LOG2(lk,
  834                                             "%s: %p claimed by a new writer",
  835                                             __func__, lk);
  836                                         break;
  837                                 }
  838                                 sleepq_release(&lk->lock_object);
  839                                 continue;
  840                         }
  841 
  842                         /*
  843                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
  844                          * fail, loop back and retry.
  845                          */
  846                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
  847                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
  848                                     x | LK_EXCLUSIVE_WAITERS)) {
  849                                         sleepq_release(&lk->lock_object);
  850                                         continue;
  851                                 }
  852                                 LOCK_LOG2(lk, "%s: %p set excl waiters flag",
  853                                     __func__, lk);
  854                         }
  855 
  856                         /*
  857                          * As far as we have been unable to acquire the
  858                          * exclusive lock and the exclusive waiters flag
  859                          * is set, we will sleep.
  860                          */
  861                         error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
  862                             SQ_EXCLUSIVE_QUEUE);
  863                         flags &= ~LK_INTERLOCK;
  864                         if (error) {
  865                                 LOCK_LOG3(lk,
  866                                     "%s: interrupted sleep for %p with %d",
  867                                     __func__, lk, error);
  868                                 break;
  869                         }
  870                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
  871                             __func__, lk);
  872                 }
  873                 if (error == 0) {
  874                         lock_profile_obtain_lock_success(&lk->lock_object,
  875                             contested, waittime, file, line);
  876                         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
  877                             lk->lk_recurse, file, line);
  878                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
  879                             LK_TRYWIT(flags), file, line);
  880                         TD_LOCKS_INC(curthread);
  881                         STACK_SAVE(lk);
  882                 }
  883                 break;
  884         case LK_DOWNGRADE:
  885                 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
  886                 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
  887                 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
  888                 TD_SLOCKS_INC(curthread);
  889 
  890                 /*
  891                  * In order to preserve waiters flags, just spin.
  892                  */
  893                 for (;;) {
  894                         x = lk->lk_lock;
  895                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
  896                         x &= LK_ALL_WAITERS;
  897                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
  898                             LK_SHARERS_LOCK(1) | x))
  899                                 break;
  900                         cpu_spinwait();
  901                 }
  902                 break;
  903         case LK_RELEASE:
  904                 _lockmgr_assert(lk, KA_LOCKED, file, line);
  905                 x = lk->lk_lock;
  906 
  907                 if ((x & LK_SHARE) == 0) {
  908 
  909                         /*
  910                          * As first option, treact the lock as if it has not
  911                          * any waiter.
  912                          * Fix-up the tid var if the lock has been disowned.
  913                          */
  914                         if (LK_HOLDER(x) == LK_KERNPROC)
  915                                 tid = LK_KERNPROC;
  916                         else {
  917                                 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE,
  918                                     file, line);
  919                                 TD_LOCKS_DEC(curthread);
  920                         }
  921                         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
  922                             lk->lk_recurse, file, line);
  923 
  924                         /*
  925                          * The lock is held in exclusive mode.
  926                          * If the lock is recursed also, then unrecurse it.
  927                          */
  928                         if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
  929                                 LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
  930                                     lk);
  931                                 lk->lk_recurse--;
  932                                 break;
  933                         }
  934                         if (tid != LK_KERNPROC)
  935                                 lock_profile_release_lock(&lk->lock_object);
  936 
  937                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
  938                             LK_UNLOCKED))
  939                                 break;
  940 
  941                         sleepq_lock(&lk->lock_object);
  942                         x = lk->lk_lock;
  943                         v = LK_UNLOCKED;
  944 
  945                         /*
  946                          * If the lock has exclusive waiters, give them
  947                          * preference in order to avoid deadlock with
  948                          * shared runners up.
  949                          * If interruptible sleeps left the exclusive queue
  950                          * empty avoid a starvation for the threads sleeping
  951                          * on the shared queue by giving them precedence
  952                          * and cleaning up the exclusive waiters bit anyway.
  953                          * Please note that the LK_EXSLPFAIL flag may be lying
  954                          * about the real presence of waiters with the
  955                          * LK_SLEEPFAIL flag on because they may be used in
  956                          * conjuction with interruptible sleeps.
  957                          */
  958                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
  959                         realexslp = sleepq_sleepcnt(&lk->lock_object,
  960                             SQ_EXCLUSIVE_QUEUE);
  961                         if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
  962                                 if ((lk->lock_object.lo_flags &
  963                                     LK_EXSLPFAIL) == 0) {
  964                                         lk->lock_object.lo_flags &=
  965                                             ~LK_EXSLPFAIL;
  966                                         queue = SQ_EXCLUSIVE_QUEUE;
  967                                         v |= (x & LK_SHARED_WAITERS);
  968                                 } else {
  969                                         lk->lock_object.lo_flags &=
  970                                             ~LK_EXSLPFAIL;
  971                                         LOCK_LOG2(lk,
  972                                         "%s: %p has only LK_SLEEPFAIL sleepers",
  973                                             __func__, lk);
  974                                         LOCK_LOG2(lk,
  975                         "%s: %p waking up threads on the exclusive queue",
  976                                             __func__, lk);
  977                                         wakeup_swapper =
  978                                             sleepq_broadcast(&lk->lock_object,
  979                                             SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
  980                                         queue = SQ_SHARED_QUEUE;
  981                                 }
  982                         } else {
  983 
  984                                 /*
  985                                  * Exclusive waiters sleeping with LK_SLEEPFAIL
  986                                  * on and using interruptible sleeps/timeout
  987                                  * may have left spourious LK_EXSLPFAIL flag
  988                                  * on, so clean it up anyway.
  989                                  */
  990                                 lk->lock_object.lo_flags &= ~LK_EXSLPFAIL;
  991                                 queue = SQ_SHARED_QUEUE;
  992                         }
  993 
  994                         LOCK_LOG3(lk,
  995                             "%s: %p waking up threads on the %s queue",
  996                             __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
  997                             "exclusive");
  998                         atomic_store_rel_ptr(&lk->lk_lock, v);
  999                         wakeup_swapper |= sleepq_broadcast(&lk->lock_object,
 1000                             SLEEPQ_LK, 0, queue);
 1001                         sleepq_release(&lk->lock_object);
 1002                         break;
 1003                 } else
 1004                         wakeup_swapper = wakeupshlk(lk, file, line);
 1005                 break;
 1006         case LK_DRAIN:
 1007                 if (LK_CAN_WITNESS(flags))
 1008                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
 1009                             LOP_EXCLUSIVE, file, line, ilk);
 1010 
 1011                 /*
 1012                  * Trying to drain a lock we already own will result in a
 1013                  * deadlock.
 1014                  */
 1015                 if (lockmgr_xlocked(lk)) {
 1016                         if (flags & LK_INTERLOCK)
 1017                                 class->lc_unlock(ilk);
 1018                         panic("%s: draining %s with the lock held @ %s:%d\n",
 1019                             __func__, iwmesg, file, line);
 1020                 }
 1021 
 1022                 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
 1023                         lock_profile_obtain_lock_failed(&lk->lock_object,
 1024                             &contested, &waittime);
 1025 
 1026                         /*
 1027                          * If the lock is expected to not sleep just give up
 1028                          * and return.
 1029                          */
 1030                         if (LK_TRYOP(flags)) {
 1031                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 1032                                     __func__, lk);
 1033                                 error = EBUSY;
 1034                                 break;
 1035                         }
 1036 
 1037                         /*
 1038                          * Acquire the sleepqueue chain lock because we
 1039                          * probabilly will need to manipulate waiters flags.
 1040                          */
 1041                         sleepq_lock(&lk->lock_object);
 1042                         x = lk->lk_lock;
 1043 
 1044                         /*
 1045                          * if the lock has been released while we spun on
 1046                          * the sleepqueue chain lock just try again.
 1047                          */
 1048                         if (x == LK_UNLOCKED) {
 1049                                 sleepq_release(&lk->lock_object);
 1050                                 continue;
 1051                         }
 1052 
 1053                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 1054                         if ((x & ~v) == LK_UNLOCKED) {
 1055                                 v = (x & ~LK_EXCLUSIVE_SPINNERS);
 1056 
 1057                                 /*
 1058                                  * If interruptible sleeps left the exclusive
 1059                                  * queue empty avoid a starvation for the
 1060                                  * threads sleeping on the shared queue by
 1061                                  * giving them precedence and cleaning up the
 1062                                  * exclusive waiters bit anyway.
 1063                                  * Please note that the LK_EXSLPFAIL flag may
 1064                                  * be lying about the real presence of waiters
 1065                                  * with the LK_SLEEPFAIL flag on because they
 1066                                  * may be used in conjuction with interruptible
 1067                                  * sleeps.
 1068                                  */
 1069                                 if (v & LK_EXCLUSIVE_WAITERS) {
 1070                                         queue = SQ_EXCLUSIVE_QUEUE;
 1071                                         v &= ~LK_EXCLUSIVE_WAITERS;
 1072                                 } else {
 1073 
 1074                                         /*
 1075                                          * Exclusive waiters sleeping with
 1076                                          * LK_SLEEPFAIL on and using
 1077                                          * interruptible sleeps/timeout may
 1078                                          * have left spourious LK_EXSLPFAIL
 1079                                          * flag on, so clean it up anyway.
 1080                                          */
 1081                                         MPASS(v & LK_SHARED_WAITERS);
 1082                                         lk->lock_object.lo_flags &=
 1083                                             ~LK_EXSLPFAIL;
 1084                                         queue = SQ_SHARED_QUEUE;
 1085                                         v &= ~LK_SHARED_WAITERS;
 1086                                 }
 1087                                 if (queue == SQ_EXCLUSIVE_QUEUE) {
 1088                                         realexslp =
 1089                                             sleepq_sleepcnt(&lk->lock_object,
 1090                                             SQ_EXCLUSIVE_QUEUE);
 1091                                         if ((lk->lock_object.lo_flags &
 1092                                             LK_EXSLPFAIL) == 0) {
 1093                                                 lk->lock_object.lo_flags &=
 1094                                                     ~LK_EXSLPFAIL;
 1095                                                 queue = SQ_SHARED_QUEUE;
 1096                                                 v &= ~LK_SHARED_WAITERS;
 1097                                                 if (realexslp != 0) {
 1098                                                         LOCK_LOG2(lk,
 1099                                         "%s: %p has only LK_SLEEPFAIL sleepers",
 1100                                                             __func__, lk);
 1101                                                         LOCK_LOG2(lk,
 1102                         "%s: %p waking up threads on the exclusive queue",
 1103                                                             __func__, lk);
 1104                                                         wakeup_swapper =
 1105                                                             sleepq_broadcast(
 1106                                                             &lk->lock_object,
 1107                                                             SLEEPQ_LK, 0,
 1108                                                             SQ_EXCLUSIVE_QUEUE);
 1109                                                 }
 1110                                         } else
 1111                                                 lk->lock_object.lo_flags &=
 1112                                                     ~LK_EXSLPFAIL;
 1113                                 }
 1114                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
 1115                                         sleepq_release(&lk->lock_object);
 1116                                         continue;
 1117                                 }
 1118                                 LOCK_LOG3(lk,
 1119                                 "%s: %p waking up all threads on the %s queue",
 1120                                     __func__, lk, queue == SQ_SHARED_QUEUE ?
 1121                                     "shared" : "exclusive");
 1122                                 wakeup_swapper |= sleepq_broadcast(
 1123                                     &lk->lock_object, SLEEPQ_LK, 0, queue);
 1124 
 1125                                 /*
 1126                                  * If shared waiters have been woken up we need
 1127                                  * to wait for one of them to acquire the lock
 1128                                  * before to set the exclusive waiters in
 1129                                  * order to avoid a deadlock.
 1130                                  */
 1131                                 if (queue == SQ_SHARED_QUEUE) {
 1132                                         for (v = lk->lk_lock;
 1133                                             (v & LK_SHARE) && !LK_SHARERS(v);
 1134                                             v = lk->lk_lock)
 1135                                                 cpu_spinwait();
 1136                                 }
 1137                         }
 1138 
 1139                         /*
 1140                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
 1141                          * fail, loop back and retry.
 1142                          */
 1143                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
 1144                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
 1145                                     x | LK_EXCLUSIVE_WAITERS)) {
 1146                                         sleepq_release(&lk->lock_object);
 1147                                         continue;
 1148                                 }
 1149                                 LOCK_LOG2(lk, "%s: %p set drain waiters flag",
 1150                                     __func__, lk);
 1151                         }
 1152 
 1153                         /*
 1154                          * As far as we have been unable to acquire the
 1155                          * exclusive lock and the exclusive waiters flag
 1156                          * is set, we will sleep.
 1157                          */
 1158                         if (flags & LK_INTERLOCK) {
 1159                                 class->lc_unlock(ilk);
 1160                                 flags &= ~LK_INTERLOCK;
 1161                         }
 1162                         GIANT_SAVE();
 1163                         sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
 1164                             SQ_EXCLUSIVE_QUEUE);
 1165                         sleepq_wait(&lk->lock_object, ipri & PRIMASK);
 1166                         GIANT_RESTORE();
 1167                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 1168                             __func__, lk);
 1169                 }
 1170 
 1171                 if (error == 0) {
 1172                         lock_profile_obtain_lock_success(&lk->lock_object,
 1173                             contested, waittime, file, line);
 1174                         LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
 1175                             lk->lk_recurse, file, line);
 1176                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
 1177                             LK_TRYWIT(flags), file, line);
 1178                         TD_LOCKS_INC(curthread);
 1179                         STACK_SAVE(lk);
 1180                 }
 1181                 break;
 1182         default:
 1183                 if (flags & LK_INTERLOCK)
 1184                         class->lc_unlock(ilk);
 1185                 panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
 1186         }
 1187 
 1188         if (flags & LK_INTERLOCK)
 1189                 class->lc_unlock(ilk);
 1190         if (wakeup_swapper)
 1191                 kick_proc0();
 1192 
 1193         return (error);
 1194 }
 1195 
 1196 void
 1197 _lockmgr_disown(struct lock *lk, const char *file, int line)
 1198 {
 1199         uintptr_t tid, x;
 1200 
 1201         tid = (uintptr_t)curthread;
 1202         _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
 1203 
 1204         /*
 1205          * If the owner is already LK_KERNPROC just skip the whole operation.
 1206          */
 1207         if (LK_HOLDER(lk->lk_lock) != tid)
 1208                 return;
 1209         lock_profile_release_lock(&lk->lock_object);
 1210         LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
 1211         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
 1212         TD_LOCKS_DEC(curthread);
 1213         STACK_SAVE(lk);
 1214 
 1215         /*
 1216          * In order to preserve waiters flags, just spin.
 1217          */
 1218         for (;;) {
 1219                 x = lk->lk_lock;
 1220                 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 1221                 x &= LK_ALL_WAITERS;
 1222                 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
 1223                     LK_KERNPROC | x))
 1224                         return;
 1225                 cpu_spinwait();
 1226         }
 1227 }
 1228 
 1229 void
 1230 lockmgr_printinfo(struct lock *lk)
 1231 {
 1232         struct thread *td;
 1233         uintptr_t x;
 1234 
 1235         if (lk->lk_lock == LK_UNLOCKED)
 1236                 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
 1237         else if (lk->lk_lock & LK_SHARE)
 1238                 printf("lock type %s: SHARED (count %ju)\n",
 1239                     lk->lock_object.lo_name,
 1240                     (uintmax_t)LK_SHARERS(lk->lk_lock));
 1241         else {
 1242                 td = lockmgr_xholder(lk);
 1243                 printf("lock type %s: EXCL by thread %p (pid %d)\n",
 1244                     lk->lock_object.lo_name, td, td->td_proc->p_pid);
 1245         }
 1246 
 1247         x = lk->lk_lock;
 1248         if (x & LK_EXCLUSIVE_WAITERS)
 1249                 printf(" with exclusive waiters pending\n");
 1250         if (x & LK_SHARED_WAITERS)
 1251                 printf(" with shared waiters pending\n");
 1252         if (x & LK_EXCLUSIVE_SPINNERS)
 1253                 printf(" with exclusive spinners pending\n");
 1254 
 1255         STACK_PRINT(lk);
 1256 }
 1257 
 1258 int
 1259 lockstatus(struct lock *lk)
 1260 {
 1261         uintptr_t v, x;
 1262         int ret;
 1263 
 1264         ret = LK_SHARED;
 1265         x = lk->lk_lock;
 1266         v = LK_HOLDER(x);
 1267 
 1268         if ((x & LK_SHARE) == 0) {
 1269                 if (v == (uintptr_t)curthread || v == LK_KERNPROC)
 1270                         ret = LK_EXCLUSIVE;
 1271                 else
 1272                         ret = LK_EXCLOTHER;
 1273         } else if (x == LK_UNLOCKED)
 1274                 ret = 0;
 1275 
 1276         return (ret);
 1277 }
 1278 
 1279 #ifdef INVARIANT_SUPPORT
 1280 #ifndef INVARIANTS
 1281 #undef  _lockmgr_assert
 1282 #endif
 1283 
 1284 void
 1285 _lockmgr_assert(struct lock *lk, int what, const char *file, int line)
 1286 {
 1287         int slocked = 0;
 1288 
 1289         if (panicstr != NULL)
 1290                 return;
 1291         switch (what) {
 1292         case KA_SLOCKED:
 1293         case KA_SLOCKED | KA_NOTRECURSED:
 1294         case KA_SLOCKED | KA_RECURSED:
 1295                 slocked = 1;
 1296         case KA_LOCKED:
 1297         case KA_LOCKED | KA_NOTRECURSED:
 1298         case KA_LOCKED | KA_RECURSED:
 1299 #ifdef WITNESS
 1300 
 1301                 /*
 1302                  * We cannot trust WITNESS if the lock is held in exclusive
 1303                  * mode and a call to lockmgr_disown() happened.
 1304                  * Workaround this skipping the check if the lock is held in
 1305                  * exclusive mode even for the KA_LOCKED case.
 1306                  */
 1307                 if (slocked || (lk->lk_lock & LK_SHARE)) {
 1308                         witness_assert(&lk->lock_object, what, file, line);
 1309                         break;
 1310                 }
 1311 #endif
 1312                 if (lk->lk_lock == LK_UNLOCKED ||
 1313                     ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
 1314                     (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
 1315                         panic("Lock %s not %slocked @ %s:%d\n",
 1316                             lk->lock_object.lo_name, slocked ? "share" : "",
 1317                             file, line);
 1318 
 1319                 if ((lk->lk_lock & LK_SHARE) == 0) {
 1320                         if (lockmgr_recursed(lk)) {
 1321                                 if (what & KA_NOTRECURSED)
 1322                                         panic("Lock %s recursed @ %s:%d\n",
 1323                                             lk->lock_object.lo_name, file,
 1324                                             line);
 1325                         } else if (what & KA_RECURSED)
 1326                                 panic("Lock %s not recursed @ %s:%d\n",
 1327                                     lk->lock_object.lo_name, file, line);
 1328                 }
 1329                 break;
 1330         case KA_XLOCKED:
 1331         case KA_XLOCKED | KA_NOTRECURSED:
 1332         case KA_XLOCKED | KA_RECURSED:
 1333                 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
 1334                         panic("Lock %s not exclusively locked @ %s:%d\n",
 1335                             lk->lock_object.lo_name, file, line);
 1336                 if (lockmgr_recursed(lk)) {
 1337                         if (what & KA_NOTRECURSED)
 1338                                 panic("Lock %s recursed @ %s:%d\n",
 1339                                     lk->lock_object.lo_name, file, line);
 1340                 } else if (what & KA_RECURSED)
 1341                         panic("Lock %s not recursed @ %s:%d\n",
 1342                             lk->lock_object.lo_name, file, line);
 1343                 break;
 1344         case KA_UNLOCKED:
 1345                 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
 1346                         panic("Lock %s exclusively locked @ %s:%d\n",
 1347                             lk->lock_object.lo_name, file, line);
 1348                 break;
 1349         default:
 1350                 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
 1351                     line);
 1352         }
 1353 }
 1354 #endif
 1355 
 1356 #ifdef DDB
 1357 int
 1358 lockmgr_chain(struct thread *td, struct thread **ownerp)
 1359 {
 1360         struct lock *lk;
 1361 
 1362         lk = td->td_wchan;
 1363 
 1364         if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
 1365                 return (0);
 1366         db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
 1367         if (lk->lk_lock & LK_SHARE)
 1368                 db_printf("SHARED (count %ju)\n",
 1369                     (uintmax_t)LK_SHARERS(lk->lk_lock));
 1370         else
 1371                 db_printf("EXCL\n");
 1372         *ownerp = lockmgr_xholder(lk);
 1373 
 1374         return (1);
 1375 }
 1376 
 1377 static void
 1378 db_show_lockmgr(struct lock_object *lock)
 1379 {
 1380         struct thread *td;
 1381         struct lock *lk;
 1382 
 1383         lk = (struct lock *)lock;
 1384 
 1385         db_printf(" state: ");
 1386         if (lk->lk_lock == LK_UNLOCKED)
 1387                 db_printf("UNLOCKED\n");
 1388         else if (lk->lk_lock & LK_SHARE)
 1389                 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
 1390         else {
 1391                 td = lockmgr_xholder(lk);
 1392                 if (td == (struct thread *)LK_KERNPROC)
 1393                         db_printf("XLOCK: LK_KERNPROC\n");
 1394                 else
 1395                         db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
 1396                             td->td_tid, td->td_proc->p_pid,
 1397                             td->td_proc->p_comm);
 1398                 if (lockmgr_recursed(lk))
 1399                         db_printf(" recursed: %d\n", lk->lk_recurse);
 1400         }
 1401         db_printf(" waiters: ");
 1402         switch (lk->lk_lock & LK_ALL_WAITERS) {
 1403         case LK_SHARED_WAITERS:
 1404                 db_printf("shared\n");
 1405                 break;
 1406         case LK_EXCLUSIVE_WAITERS:
 1407                 db_printf("exclusive\n");
 1408                 break;
 1409         case LK_ALL_WAITERS:
 1410                 db_printf("shared and exclusive\n");
 1411                 break;
 1412         default:
 1413                 db_printf("none\n");
 1414         }
 1415         db_printf(" spinners: ");
 1416         if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
 1417                 db_printf("exclusive\n");
 1418         else
 1419                 db_printf("none\n");
 1420 }
 1421 #endif

Cache object: e25b162ff15a58af137503b0057e6345


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.