The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_lock.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice(s), this list of conditions and the following disclaimer as
   12  *    the first lines of this file unmodified other than the possible
   13  *    addition of one or more copyright notices.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice(s), this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
   19  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   21  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
   22  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   23  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   25  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   28  * DAMAGE.
   29  */
   30 
   31 #include "opt_ddb.h"
   32 #include "opt_hwpmc_hooks.h"
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/12.0/sys/kern/kern_lock.c 336267 2018-07-13 22:40:14Z mjg $");
   36 
   37 #include <sys/param.h>
   38 #include <sys/kdb.h>
   39 #include <sys/ktr.h>
   40 #include <sys/lock.h>
   41 #include <sys/lock_profile.h>
   42 #include <sys/lockmgr.h>
   43 #include <sys/mutex.h>
   44 #include <sys/proc.h>
   45 #include <sys/sleepqueue.h>
   46 #ifdef DEBUG_LOCKS
   47 #include <sys/stack.h>
   48 #endif
   49 #include <sys/sysctl.h>
   50 #include <sys/systm.h>
   51 
   52 #include <machine/cpu.h>
   53 
   54 #ifdef DDB
   55 #include <ddb/ddb.h>
   56 #endif
   57 
   58 #ifdef HWPMC_HOOKS
   59 #include <sys/pmckern.h>
   60 PMC_SOFT_DECLARE( , , lock, failed);
   61 #endif
   62 
   63 CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
   64     (LK_ADAPTIVE | LK_NOSHARE));
   65 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
   66     ~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
   67 
   68 #define SQ_EXCLUSIVE_QUEUE      0
   69 #define SQ_SHARED_QUEUE         1
   70 
   71 #ifndef INVARIANTS
   72 #define _lockmgr_assert(lk, what, file, line)
   73 #endif
   74 
   75 #define TD_SLOCKS_INC(td)       ((td)->td_lk_slocks++)
   76 #define TD_SLOCKS_DEC(td)       ((td)->td_lk_slocks--)
   77 
   78 #ifndef DEBUG_LOCKS
   79 #define STACK_PRINT(lk)
   80 #define STACK_SAVE(lk)
   81 #define STACK_ZERO(lk)
   82 #else
   83 #define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
   84 #define STACK_SAVE(lk)  stack_save(&(lk)->lk_stack)
   85 #define STACK_ZERO(lk)  stack_zero(&(lk)->lk_stack)
   86 #endif
   87 
   88 #define LOCK_LOG2(lk, string, arg1, arg2)                               \
   89         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
   90                 CTR2(KTR_LOCK, (string), (arg1), (arg2))
   91 #define LOCK_LOG3(lk, string, arg1, arg2, arg3)                         \
   92         if (LOCK_LOG_TEST(&(lk)->lock_object, 0))                       \
   93                 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
   94 
   95 #define GIANT_DECLARE                                                   \
   96         int _i = 0;                                                     \
   97         WITNESS_SAVE_DECL(Giant)
   98 #define GIANT_RESTORE() do {                                            \
   99         if (_i > 0) {                                                   \
  100                 while (_i--)                                            \
  101                         mtx_lock(&Giant);                               \
  102                 WITNESS_RESTORE(&Giant.lock_object, Giant);             \
  103         }                                                               \
  104 } while (0)
  105 #define GIANT_SAVE() do {                                               \
  106         if (mtx_owned(&Giant)) {                                        \
  107                 WITNESS_SAVE(&Giant.lock_object, Giant);                \
  108                 while (mtx_owned(&Giant)) {                             \
  109                         _i++;                                           \
  110                         mtx_unlock(&Giant);                             \
  111                 }                                                       \
  112         }                                                               \
  113 } while (0)
  114 
  115 static bool __always_inline
  116 LK_CAN_SHARE(uintptr_t x, int flags, bool fp)
  117 {
  118 
  119         if ((x & (LK_SHARE | LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) ==
  120             LK_SHARE)
  121                 return (true);
  122         if (fp || (!(x & LK_SHARE)))
  123                 return (false);
  124         if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
  125             (curthread->td_pflags & TDP_DEADLKTREAT))
  126                 return (true);
  127         return (false);
  128 }
  129 
  130 #define LK_TRYOP(x)                                                     \
  131         ((x) & LK_NOWAIT)
  132 
  133 #define LK_CAN_WITNESS(x)                                               \
  134         (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
  135 #define LK_TRYWIT(x)                                                    \
  136         (LK_TRYOP(x) ? LOP_TRYLOCK : 0)
  137 
  138 #define LK_CAN_ADAPT(lk, f)                                             \
  139         (((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 &&             \
  140         ((f) & LK_SLEEPFAIL) == 0)
  141 
  142 #define lockmgr_disowned(lk)                                            \
  143         (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
  144 
  145 #define lockmgr_xlocked_v(v)                                            \
  146         (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
  147 
  148 #define lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock)
  149 
  150 static void     assert_lockmgr(const struct lock_object *lock, int how);
  151 #ifdef DDB
  152 static void     db_show_lockmgr(const struct lock_object *lock);
  153 #endif
  154 static void     lock_lockmgr(struct lock_object *lock, uintptr_t how);
  155 #ifdef KDTRACE_HOOKS
  156 static int      owner_lockmgr(const struct lock_object *lock,
  157                     struct thread **owner);
  158 #endif
  159 static uintptr_t unlock_lockmgr(struct lock_object *lock);
  160 
  161 struct lock_class lock_class_lockmgr = {
  162         .lc_name = "lockmgr",
  163         .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
  164         .lc_assert = assert_lockmgr,
  165 #ifdef DDB
  166         .lc_ddb_show = db_show_lockmgr,
  167 #endif
  168         .lc_lock = lock_lockmgr,
  169         .lc_unlock = unlock_lockmgr,
  170 #ifdef KDTRACE_HOOKS
  171         .lc_owner = owner_lockmgr,
  172 #endif
  173 };
  174 
  175 struct lockmgr_wait {
  176         const char *iwmesg;
  177         int ipri;
  178         int itimo;
  179 };
  180 
  181 static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
  182     int flags, bool fp);
  183 static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp);
  184 
  185 static void
  186 lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper)
  187 {
  188         struct lock_class *class;
  189 
  190         if (flags & LK_INTERLOCK) {
  191                 class = LOCK_CLASS(ilk);
  192                 class->lc_unlock(ilk);
  193         }
  194 
  195         if (__predict_false(wakeup_swapper))
  196                 kick_proc0();
  197 }
  198 
  199 static void
  200 lockmgr_note_shared_acquire(struct lock *lk, int contested,
  201     uint64_t waittime, const char *file, int line, int flags)
  202 {
  203 
  204         lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
  205             file, line);
  206         LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
  207         WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
  208         TD_LOCKS_INC(curthread);
  209         TD_SLOCKS_INC(curthread);
  210         STACK_SAVE(lk);
  211 }
  212 
  213 static void
  214 lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
  215 {
  216 
  217         lock_profile_release_lock(&lk->lock_object);
  218         WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
  219         LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
  220         TD_LOCKS_DEC(curthread);
  221         TD_SLOCKS_DEC(curthread);
  222 }
  223 
  224 static void
  225 lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
  226     uint64_t waittime, const char *file, int line, int flags)
  227 {
  228 
  229         lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,
  230             file, line);
  231         LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
  232         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
  233             line);
  234         TD_LOCKS_INC(curthread);
  235         STACK_SAVE(lk);
  236 }
  237 
  238 static void
  239 lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
  240 {
  241 
  242         lock_profile_release_lock(&lk->lock_object);
  243         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
  244             line);
  245         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
  246         TD_LOCKS_DEC(curthread);
  247 }
  248 
  249 static __inline struct thread *
  250 lockmgr_xholder(const struct lock *lk)
  251 {
  252         uintptr_t x;
  253 
  254         x = lk->lk_lock;
  255         return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
  256 }
  257 
  258 /*
  259  * It assumes sleepq_lock held and returns with this one unheld.
  260  * It also assumes the generic interlock is sane and previously checked.
  261  * If LK_INTERLOCK is specified the interlock is not reacquired after the
  262  * sleep.
  263  */
  264 static __inline int
  265 sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
  266     const char *wmesg, int pri, int timo, int queue)
  267 {
  268         GIANT_DECLARE;
  269         struct lock_class *class;
  270         int catch, error;
  271 
  272         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
  273         catch = pri & PCATCH;
  274         pri &= PRIMASK;
  275         error = 0;
  276 
  277         LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
  278             (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
  279 
  280         if (flags & LK_INTERLOCK)
  281                 class->lc_unlock(ilk);
  282         if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
  283                 lk->lk_exslpfail++;
  284         GIANT_SAVE();
  285         sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
  286             SLEEPQ_INTERRUPTIBLE : 0), queue);
  287         if ((flags & LK_TIMELOCK) && timo)
  288                 sleepq_set_timeout(&lk->lock_object, timo);
  289 
  290         /*
  291          * Decisional switch for real sleeping.
  292          */
  293         if ((flags & LK_TIMELOCK) && timo && catch)
  294                 error = sleepq_timedwait_sig(&lk->lock_object, pri);
  295         else if ((flags & LK_TIMELOCK) && timo)
  296                 error = sleepq_timedwait(&lk->lock_object, pri);
  297         else if (catch)
  298                 error = sleepq_wait_sig(&lk->lock_object, pri);
  299         else
  300                 sleepq_wait(&lk->lock_object, pri);
  301         GIANT_RESTORE();
  302         if ((flags & LK_SLEEPFAIL) && error == 0)
  303                 error = ENOLCK;
  304 
  305         return (error);
  306 }
  307 
  308 static __inline int
  309 wakeupshlk(struct lock *lk, const char *file, int line)
  310 {
  311         uintptr_t v, x, orig_x;
  312         u_int realexslp;
  313         int queue, wakeup_swapper;
  314 
  315         wakeup_swapper = 0;
  316         for (;;) {
  317                 x = lk->lk_lock;
  318                 if (lockmgr_sunlock_try(lk, &x))
  319                         break;
  320 
  321                 /*
  322                  * We should have a sharer with waiters, so enter the hard
  323                  * path in order to handle wakeups correctly.
  324                  */
  325                 sleepq_lock(&lk->lock_object);
  326                 orig_x = lk->lk_lock;
  327 retry_sleepq:
  328                 x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
  329                 v = LK_UNLOCKED;
  330 
  331                 /*
  332                  * If the lock has exclusive waiters, give them preference in
  333                  * order to avoid deadlock with shared runners up.
  334                  * If interruptible sleeps left the exclusive queue empty
  335                  * avoid a starvation for the threads sleeping on the shared
  336                  * queue by giving them precedence and cleaning up the
  337                  * exclusive waiters bit anyway.
  338                  * Please note that lk_exslpfail count may be lying about
  339                  * the real number of waiters with the LK_SLEEPFAIL flag on
  340                  * because they may be used in conjunction with interruptible
  341                  * sleeps so lk_exslpfail might be considered an 'upper limit'
  342                  * bound, including the edge cases.
  343                  */
  344                 realexslp = sleepq_sleepcnt(&lk->lock_object,
  345                     SQ_EXCLUSIVE_QUEUE);
  346                 if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
  347                         if (lk->lk_exslpfail < realexslp) {
  348                                 lk->lk_exslpfail = 0;
  349                                 queue = SQ_EXCLUSIVE_QUEUE;
  350                                 v |= (x & LK_SHARED_WAITERS);
  351                         } else {
  352                                 lk->lk_exslpfail = 0;
  353                                 LOCK_LOG2(lk,
  354                                     "%s: %p has only LK_SLEEPFAIL sleepers",
  355                                     __func__, lk);
  356                                 LOCK_LOG2(lk,
  357                             "%s: %p waking up threads on the exclusive queue",
  358                                     __func__, lk);
  359                                 wakeup_swapper =
  360                                     sleepq_broadcast(&lk->lock_object,
  361                                     SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
  362                                 queue = SQ_SHARED_QUEUE;
  363                         }
  364                                 
  365                 } else {
  366 
  367                         /*
  368                          * Exclusive waiters sleeping with LK_SLEEPFAIL on
  369                          * and using interruptible sleeps/timeout may have
  370                          * left spourious lk_exslpfail counts on, so clean
  371                          * it up anyway.
  372                          */
  373                         lk->lk_exslpfail = 0;
  374                         queue = SQ_SHARED_QUEUE;
  375                 }
  376 
  377                 if (lockmgr_sunlock_try(lk, &orig_x)) {
  378                         sleepq_release(&lk->lock_object);
  379                         break;
  380                 }
  381 
  382                 x |= LK_SHARERS_LOCK(1);
  383                 if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
  384                         orig_x = x;
  385                         goto retry_sleepq;
  386                 }
  387                 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
  388                     __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
  389                     "exclusive");
  390                 wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
  391                     0, queue);
  392                 sleepq_release(&lk->lock_object);
  393                 break;
  394         }
  395 
  396         lockmgr_note_shared_release(lk, file, line);
  397         return (wakeup_swapper);
  398 }
  399 
  400 static void
  401 assert_lockmgr(const struct lock_object *lock, int what)
  402 {
  403 
  404         panic("lockmgr locks do not support assertions");
  405 }
  406 
  407 static void
  408 lock_lockmgr(struct lock_object *lock, uintptr_t how)
  409 {
  410 
  411         panic("lockmgr locks do not support sleep interlocking");
  412 }
  413 
  414 static uintptr_t
  415 unlock_lockmgr(struct lock_object *lock)
  416 {
  417 
  418         panic("lockmgr locks do not support sleep interlocking");
  419 }
  420 
  421 #ifdef KDTRACE_HOOKS
  422 static int
  423 owner_lockmgr(const struct lock_object *lock, struct thread **owner)
  424 {
  425 
  426         panic("lockmgr locks do not support owner inquiring");
  427 }
  428 #endif
  429 
  430 void
  431 lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
  432 {
  433         int iflags;
  434 
  435         MPASS((flags & ~LK_INIT_MASK) == 0);
  436         ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
  437             ("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
  438             &lk->lk_lock));
  439 
  440         iflags = LO_SLEEPABLE | LO_UPGRADABLE;
  441         if (flags & LK_CANRECURSE)
  442                 iflags |= LO_RECURSABLE;
  443         if ((flags & LK_NODUP) == 0)
  444                 iflags |= LO_DUPOK;
  445         if (flags & LK_NOPROFILE)
  446                 iflags |= LO_NOPROFILE;
  447         if ((flags & LK_NOWITNESS) == 0)
  448                 iflags |= LO_WITNESS;
  449         if (flags & LK_QUIET)
  450                 iflags |= LO_QUIET;
  451         if (flags & LK_IS_VNODE)
  452                 iflags |= LO_IS_VNODE;
  453         iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
  454 
  455         lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
  456         lk->lk_lock = LK_UNLOCKED;
  457         lk->lk_recurse = 0;
  458         lk->lk_exslpfail = 0;
  459         lk->lk_timo = timo;
  460         lk->lk_pri = pri;
  461         STACK_ZERO(lk);
  462 }
  463 
  464 /*
  465  * XXX: Gross hacks to manipulate external lock flags after
  466  * initialization.  Used for certain vnode and buf locks.
  467  */
  468 void
  469 lockallowshare(struct lock *lk)
  470 {
  471 
  472         lockmgr_assert(lk, KA_XLOCKED);
  473         lk->lock_object.lo_flags &= ~LK_NOSHARE;
  474 }
  475 
  476 void
  477 lockdisableshare(struct lock *lk)
  478 {
  479 
  480         lockmgr_assert(lk, KA_XLOCKED);
  481         lk->lock_object.lo_flags |= LK_NOSHARE;
  482 }
  483 
  484 void
  485 lockallowrecurse(struct lock *lk)
  486 {
  487 
  488         lockmgr_assert(lk, KA_XLOCKED);
  489         lk->lock_object.lo_flags |= LO_RECURSABLE;
  490 }
  491 
  492 void
  493 lockdisablerecurse(struct lock *lk)
  494 {
  495 
  496         lockmgr_assert(lk, KA_XLOCKED);
  497         lk->lock_object.lo_flags &= ~LO_RECURSABLE;
  498 }
  499 
  500 void
  501 lockdestroy(struct lock *lk)
  502 {
  503 
  504         KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
  505         KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
  506         KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
  507         lock_destroy(&lk->lock_object);
  508 }
  509 
  510 static bool __always_inline
  511 lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp)
  512 {
  513 
  514         /*
  515          * If no other thread has an exclusive lock, or
  516          * no exclusive waiter is present, bump the count of
  517          * sharers.  Since we have to preserve the state of
  518          * waiters, if we fail to acquire the shared lock
  519          * loop back and retry.
  520          */
  521         *xp = lk->lk_lock;
  522         while (LK_CAN_SHARE(*xp, flags, fp)) {
  523                 if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
  524                     *xp + LK_ONE_SHARER)) {
  525                         return (true);
  526                 }
  527         }
  528         return (false);
  529 }
  530 
  531 static bool __always_inline
  532 lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
  533 {
  534 
  535         for (;;) {
  536                 if (LK_SHARERS(*xp) > 1 || !(*xp & LK_ALL_WAITERS)) {
  537                         if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
  538                             *xp - LK_ONE_SHARER))
  539                                 return (true);
  540                         continue;
  541                 }
  542                 break;
  543         }
  544         return (false);
  545 }
  546 
  547 static __noinline int
  548 lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
  549     const char *file, int line, struct lockmgr_wait *lwa)
  550 {
  551         uintptr_t tid, x;
  552         int error = 0;
  553         const char *iwmesg;
  554         int ipri, itimo;
  555 
  556 #ifdef LOCK_PROFILING
  557         uint64_t waittime = 0;
  558         int contested = 0;
  559 #endif
  560 
  561         if (__predict_false(panicstr != NULL))
  562                 goto out;
  563 
  564         tid = (uintptr_t)curthread;
  565 
  566         if (LK_CAN_WITNESS(flags))
  567                 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
  568                     file, line, flags & LK_INTERLOCK ? ilk : NULL);
  569         for (;;) {
  570                 if (lockmgr_slock_try(lk, &x, flags, false))
  571                         break;
  572 #ifdef HWPMC_HOOKS
  573                 PMC_SOFT_CALL( , , lock, failed);
  574 #endif
  575                 lock_profile_obtain_lock_failed(&lk->lock_object,
  576                     &contested, &waittime);
  577 
  578                 /*
  579                  * If the lock is already held by curthread in
  580                  * exclusive way avoid a deadlock.
  581                  */
  582                 if (LK_HOLDER(x) == tid) {
  583                         LOCK_LOG2(lk,
  584                             "%s: %p already held in exclusive mode",
  585                             __func__, lk);
  586                         error = EDEADLK;
  587                         break;
  588                 }
  589 
  590                 /*
  591                  * If the lock is expected to not sleep just give up
  592                  * and return.
  593                  */
  594                 if (LK_TRYOP(flags)) {
  595                         LOCK_LOG2(lk, "%s: %p fails the try operation",
  596                             __func__, lk);
  597                         error = EBUSY;
  598                         break;
  599                 }
  600 
  601                 /*
  602                  * Acquire the sleepqueue chain lock because we
  603                  * probabilly will need to manipulate waiters flags.
  604                  */
  605                 sleepq_lock(&lk->lock_object);
  606                 x = lk->lk_lock;
  607 retry_sleepq:
  608 
  609                 /*
  610                  * if the lock can be acquired in shared mode, try
  611                  * again.
  612                  */
  613                 if (LK_CAN_SHARE(x, flags, false)) {
  614                         sleepq_release(&lk->lock_object);
  615                         continue;
  616                 }
  617 
  618                 /*
  619                  * Try to set the LK_SHARED_WAITERS flag.  If we fail,
  620                  * loop back and retry.
  621                  */
  622                 if ((x & LK_SHARED_WAITERS) == 0) {
  623                         if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
  624                             x | LK_SHARED_WAITERS)) {
  625                                 goto retry_sleepq;
  626                         }
  627                         LOCK_LOG2(lk, "%s: %p set shared waiters flag",
  628                             __func__, lk);
  629                 }
  630 
  631                 if (lwa == NULL) {
  632                         iwmesg = lk->lock_object.lo_name;
  633                         ipri = lk->lk_pri;
  634                         itimo = lk->lk_timo;
  635                 } else {
  636                         iwmesg = lwa->iwmesg;
  637                         ipri = lwa->ipri;
  638                         itimo = lwa->itimo;
  639                 }
  640 
  641                 /*
  642                  * As far as we have been unable to acquire the
  643                  * shared lock and the shared waiters flag is set,
  644                  * we will sleep.
  645                  */
  646                 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
  647                     SQ_SHARED_QUEUE);
  648                 flags &= ~LK_INTERLOCK;
  649                 if (error) {
  650                         LOCK_LOG3(lk,
  651                             "%s: interrupted sleep for %p with %d",
  652                             __func__, lk, error);
  653                         break;
  654                 }
  655                 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
  656                     __func__, lk);
  657         }
  658         if (error == 0) {
  659 #ifdef LOCK_PROFILING
  660                 lockmgr_note_shared_acquire(lk, contested, waittime,
  661                     file, line, flags);
  662 #else
  663                 lockmgr_note_shared_acquire(lk, 0, 0, file, line,
  664                     flags);
  665 #endif
  666         }
  667 
  668 out:
  669         lockmgr_exit(flags, ilk, 0);
  670         return (error);
  671 }
  672 
  673 static __noinline int
  674 lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
  675     const char *file, int line, struct lockmgr_wait *lwa)
  676 {
  677         struct lock_class *class;
  678         uintptr_t tid, x, v;
  679         int error = 0;
  680         const char *iwmesg;
  681         int ipri, itimo;
  682 
  683 #ifdef LOCK_PROFILING
  684         uint64_t waittime = 0;
  685         int contested = 0;
  686 #endif
  687 
  688         if (__predict_false(panicstr != NULL))
  689                 goto out;
  690 
  691         tid = (uintptr_t)curthread;
  692 
  693         if (LK_CAN_WITNESS(flags))
  694                 WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
  695                     LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
  696                     ilk : NULL);
  697 
  698         /*
  699          * If curthread already holds the lock and this one is
  700          * allowed to recurse, simply recurse on it.
  701          */
  702         if (lockmgr_xlocked(lk)) {
  703                 if ((flags & LK_CANRECURSE) == 0 &&
  704                     (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
  705                         /*
  706                          * If the lock is expected to not panic just
  707                          * give up and return.
  708                          */
  709                         if (LK_TRYOP(flags)) {
  710                                 LOCK_LOG2(lk,
  711                                     "%s: %p fails the try operation",
  712                                     __func__, lk);
  713                                 error = EBUSY;
  714                                 goto out;
  715                         }
  716                         if (flags & LK_INTERLOCK) {
  717                                 class = LOCK_CLASS(ilk);
  718                                 class->lc_unlock(ilk);
  719                         }
  720                         panic("%s: recursing on non recursive lockmgr %p "
  721                             "@ %s:%d\n", __func__, lk, file, line);
  722                 }
  723                 lk->lk_recurse++;
  724                 LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
  725                 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
  726                     lk->lk_recurse, file, line);
  727                 WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
  728                     LK_TRYWIT(flags), file, line);
  729                 TD_LOCKS_INC(curthread);
  730                 goto out;
  731         }
  732 
  733         for (;;) {
  734                 if (lk->lk_lock == LK_UNLOCKED &&
  735                     atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
  736                         break;
  737 #ifdef HWPMC_HOOKS
  738                 PMC_SOFT_CALL( , , lock, failed);
  739 #endif
  740                 lock_profile_obtain_lock_failed(&lk->lock_object,
  741                     &contested, &waittime);
  742 
  743                 /*
  744                  * If the lock is expected to not sleep just give up
  745                  * and return.
  746                  */
  747                 if (LK_TRYOP(flags)) {
  748                         LOCK_LOG2(lk, "%s: %p fails the try operation",
  749                             __func__, lk);
  750                         error = EBUSY;
  751                         break;
  752                 }
  753 
  754                 /*
  755                  * Acquire the sleepqueue chain lock because we
  756                  * probabilly will need to manipulate waiters flags.
  757                  */
  758                 sleepq_lock(&lk->lock_object);
  759                 x = lk->lk_lock;
  760 retry_sleepq:
  761 
  762                 /*
  763                  * if the lock has been released while we spun on
  764                  * the sleepqueue chain lock just try again.
  765                  */
  766                 if (x == LK_UNLOCKED) {
  767                         sleepq_release(&lk->lock_object);
  768                         continue;
  769                 }
  770 
  771                 /*
  772                  * The lock can be in the state where there is a
  773                  * pending queue of waiters, but still no owner.
  774                  * This happens when the lock is contested and an
  775                  * owner is going to claim the lock.
  776                  * If curthread is the one successfully acquiring it
  777                  * claim lock ownership and return, preserving waiters
  778                  * flags.
  779                  */
  780                 v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
  781                 if ((x & ~v) == LK_UNLOCKED) {
  782                         v &= ~LK_EXCLUSIVE_SPINNERS;
  783                         if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
  784                             tid | v)) {
  785                                 sleepq_release(&lk->lock_object);
  786                                 LOCK_LOG2(lk,
  787                                     "%s: %p claimed by a new writer",
  788                                     __func__, lk);
  789                                 break;
  790                         }
  791                         goto retry_sleepq;
  792                 }
  793 
  794                 /*
  795                  * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
  796                  * fail, loop back and retry.
  797                  */
  798                 if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
  799                         if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
  800                             x | LK_EXCLUSIVE_WAITERS)) {
  801                                 goto retry_sleepq;
  802                         }
  803                         LOCK_LOG2(lk, "%s: %p set excl waiters flag",
  804                             __func__, lk);
  805                 }
  806 
  807                 if (lwa == NULL) {
  808                         iwmesg = lk->lock_object.lo_name;
  809                         ipri = lk->lk_pri;
  810                         itimo = lk->lk_timo;
  811                 } else {
  812                         iwmesg = lwa->iwmesg;
  813                         ipri = lwa->ipri;
  814                         itimo = lwa->itimo;
  815                 }
  816 
  817                 /*
  818                  * As far as we have been unable to acquire the
  819                  * exclusive lock and the exclusive waiters flag
  820                  * is set, we will sleep.
  821                  */
  822                 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
  823                     SQ_EXCLUSIVE_QUEUE);
  824                 flags &= ~LK_INTERLOCK;
  825                 if (error) {
  826                         LOCK_LOG3(lk,
  827                             "%s: interrupted sleep for %p with %d",
  828                             __func__, lk, error);
  829                         break;
  830                 }
  831                 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
  832                     __func__, lk);
  833         }
  834         if (error == 0) {
  835 #ifdef LOCK_PROFILING
  836                 lockmgr_note_exclusive_acquire(lk, contested, waittime,
  837                     file, line, flags);
  838 #else
  839                 lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
  840                     flags);
  841 #endif
  842         }
  843 
  844 out:
  845         lockmgr_exit(flags, ilk, 0);
  846         return (error);
  847 }
  848 
  849 static __noinline int
  850 lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
  851     const char *file, int line, struct lockmgr_wait *lwa)
  852 {
  853         uintptr_t tid, x, v;
  854         int error = 0;
  855         int wakeup_swapper = 0;
  856         int op;
  857 
  858         if (__predict_false(panicstr != NULL))
  859                 goto out;
  860 
  861         tid = (uintptr_t)curthread;
  862 
  863         _lockmgr_assert(lk, KA_SLOCKED, file, line);
  864         v = lk->lk_lock;
  865         x = v & LK_ALL_WAITERS;
  866         v &= LK_EXCLUSIVE_SPINNERS;
  867 
  868         /*
  869          * Try to switch from one shared lock to an exclusive one.
  870          * We need to preserve waiters flags during the operation.
  871          */
  872         if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
  873             tid | x)) {
  874                 LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
  875                     line);
  876                 WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
  877                     LK_TRYWIT(flags), file, line);
  878                 TD_SLOCKS_DEC(curthread);
  879                 goto out;
  880         }
  881 
  882         op = flags & LK_TYPE_MASK;
  883 
  884         /*
  885          * In LK_TRYUPGRADE mode, do not drop the lock,
  886          * returning EBUSY instead.
  887          */
  888         if (op == LK_TRYUPGRADE) {
  889                 LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
  890                     __func__, lk);
  891                 error = EBUSY;
  892                 goto out;
  893         }
  894 
  895         /*
  896          * We have been unable to succeed in upgrading, so just
  897          * give up the shared lock.
  898          */
  899         wakeup_swapper |= wakeupshlk(lk, file, line);
  900         error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
  901         flags &= ~LK_INTERLOCK;
  902 out:
  903         lockmgr_exit(flags, ilk, wakeup_swapper);
  904         return (error);
  905 }
  906 
  907 int
  908 lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
  909     const char *file, int line)
  910 {
  911         struct lock_class *class;
  912         uintptr_t x, tid;
  913         u_int op;
  914         bool locked;
  915 
  916         if (__predict_false(panicstr != NULL))
  917                 return (0);
  918 
  919         op = flags & LK_TYPE_MASK;
  920         locked = false;
  921         switch (op) {
  922         case LK_SHARED:
  923                 if (LK_CAN_WITNESS(flags))
  924                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
  925                             file, line, flags & LK_INTERLOCK ? ilk : NULL);
  926                 if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
  927                         break;
  928                 if (lockmgr_slock_try(lk, &x, flags, true)) {
  929                         lockmgr_note_shared_acquire(lk, 0, 0,
  930                             file, line, flags);
  931                         locked = true;
  932                 } else {
  933                         return (lockmgr_slock_hard(lk, flags, ilk, file, line,
  934                             NULL));
  935                 }
  936                 break;
  937         case LK_EXCLUSIVE:
  938                 if (LK_CAN_WITNESS(flags))
  939                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
  940                             LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
  941                             ilk : NULL);
  942                 tid = (uintptr_t)curthread;
  943                 if (lk->lk_lock == LK_UNLOCKED &&
  944                     atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
  945                         lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
  946                             flags);
  947                         locked = true;
  948                 } else {
  949                         return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
  950                             NULL));
  951                 }
  952                 break;
  953         case LK_UPGRADE:
  954         case LK_TRYUPGRADE:
  955                 return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
  956         default:
  957                 break;
  958         }
  959         if (__predict_true(locked)) {
  960                 if (__predict_false(flags & LK_INTERLOCK)) {
  961                         class = LOCK_CLASS(ilk);
  962                         class->lc_unlock(ilk);
  963                 }
  964                 return (0);
  965         } else {
  966                 return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
  967                     LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
  968         }
  969 }
  970 
  971 static __noinline int
  972 lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
  973     const char *file, int line)
  974 
  975 {
  976         int wakeup_swapper = 0;
  977 
  978         if (__predict_false(panicstr != NULL))
  979                 goto out;
  980 
  981         wakeup_swapper = wakeupshlk(lk, file, line);
  982 
  983 out:
  984         lockmgr_exit(flags, ilk, wakeup_swapper);
  985         return (0);
  986 }
  987 
  988 static __noinline int
  989 lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
  990     const char *file, int line)
  991 {
  992         uintptr_t tid, v;
  993         int wakeup_swapper = 0;
  994         u_int realexslp;
  995         int queue;
  996 
  997         if (__predict_false(panicstr != NULL))
  998                 goto out;
  999 
 1000         tid = (uintptr_t)curthread;
 1001 
 1002         /*
 1003          * As first option, treact the lock as if it has not
 1004          * any waiter.
 1005          * Fix-up the tid var if the lock has been disowned.
 1006          */
 1007         if (LK_HOLDER(x) == LK_KERNPROC)
 1008                 tid = LK_KERNPROC;
 1009         else {
 1010                 WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
 1011                 TD_LOCKS_DEC(curthread);
 1012         }
 1013         LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
 1014 
 1015         /*
 1016          * The lock is held in exclusive mode.
 1017          * If the lock is recursed also, then unrecurse it.
 1018          */
 1019         if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {
 1020                 LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
 1021                 lk->lk_recurse--;
 1022                 goto out;
 1023         }
 1024         if (tid != LK_KERNPROC)
 1025                 lock_profile_release_lock(&lk->lock_object);
 1026 
 1027         if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
 1028                 goto out;
 1029 
 1030         sleepq_lock(&lk->lock_object);
 1031         x = lk->lk_lock;
 1032         v = LK_UNLOCKED;
 1033 
 1034         /*
 1035          * If the lock has exclusive waiters, give them
 1036          * preference in order to avoid deadlock with
 1037          * shared runners up.
 1038          * If interruptible sleeps left the exclusive queue
 1039          * empty avoid a starvation for the threads sleeping
 1040          * on the shared queue by giving them precedence
 1041          * and cleaning up the exclusive waiters bit anyway.
 1042          * Please note that lk_exslpfail count may be lying
 1043          * about the real number of waiters with the
 1044          * LK_SLEEPFAIL flag on because they may be used in
 1045          * conjunction with interruptible sleeps so
 1046          * lk_exslpfail might be considered an 'upper limit'
 1047          * bound, including the edge cases.
 1048          */
 1049         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 1050         realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
 1051         if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
 1052                 if (lk->lk_exslpfail < realexslp) {
 1053                         lk->lk_exslpfail = 0;
 1054                         queue = SQ_EXCLUSIVE_QUEUE;
 1055                         v |= (x & LK_SHARED_WAITERS);
 1056                 } else {
 1057                         lk->lk_exslpfail = 0;
 1058                         LOCK_LOG2(lk,
 1059                             "%s: %p has only LK_SLEEPFAIL sleepers",
 1060                             __func__, lk);
 1061                         LOCK_LOG2(lk,
 1062                             "%s: %p waking up threads on the exclusive queue",
 1063                             __func__, lk);
 1064                         wakeup_swapper = sleepq_broadcast(&lk->lock_object,
 1065                             SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
 1066                         queue = SQ_SHARED_QUEUE;
 1067                 }
 1068         } else {
 1069 
 1070                 /*
 1071                  * Exclusive waiters sleeping with LK_SLEEPFAIL
 1072                  * on and using interruptible sleeps/timeout
 1073                  * may have left spourious lk_exslpfail counts
 1074                  * on, so clean it up anyway.
 1075                  */
 1076                 lk->lk_exslpfail = 0;
 1077                 queue = SQ_SHARED_QUEUE;
 1078         }
 1079 
 1080         LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
 1081             __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
 1082             "exclusive");
 1083         atomic_store_rel_ptr(&lk->lk_lock, v);
 1084         wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
 1085         sleepq_release(&lk->lock_object);
 1086 
 1087 out:
 1088         lockmgr_exit(flags, ilk, wakeup_swapper);
 1089         return (0);
 1090 }
 1091 
 1092 int
 1093 lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
 1094 {
 1095         struct lock_class *class;
 1096         uintptr_t x, tid;
 1097         const char *file;
 1098         int line;
 1099 
 1100         if (__predict_false(panicstr != NULL))
 1101                 return (0);
 1102 
 1103         file = __FILE__;
 1104         line = __LINE__;
 1105 
 1106         _lockmgr_assert(lk, KA_LOCKED, file, line);
 1107         x = lk->lk_lock;
 1108         if (__predict_true(x & LK_SHARE) != 0) {
 1109                 if (lockmgr_sunlock_try(lk, &x)) {
 1110                         lockmgr_note_shared_release(lk, file, line);
 1111                 } else {
 1112                         return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
 1113                 }
 1114         } else {
 1115                 tid = (uintptr_t)curthread;
 1116                 if (!lockmgr_recursed(lk) &&
 1117                     atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
 1118                         lockmgr_note_exclusive_release(lk, file, line);
 1119                 } else {
 1120                         return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
 1121                 }
 1122         }
 1123         if (__predict_false(flags & LK_INTERLOCK)) {
 1124                 class = LOCK_CLASS(ilk);
 1125                 class->lc_unlock(ilk);
 1126         }
 1127         return (0);
 1128 }
 1129 
 1130 int
 1131 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 1132     const char *wmesg, int pri, int timo, const char *file, int line)
 1133 {
 1134         GIANT_DECLARE;
 1135         struct lockmgr_wait lwa;
 1136         struct lock_class *class;
 1137         const char *iwmesg;
 1138         uintptr_t tid, v, x;
 1139         u_int op, realexslp;
 1140         int error, ipri, itimo, queue, wakeup_swapper;
 1141 #ifdef LOCK_PROFILING
 1142         uint64_t waittime = 0;
 1143         int contested = 0;
 1144 #endif
 1145 
 1146         if (panicstr != NULL)
 1147                 return (0);
 1148 
 1149         error = 0;
 1150         tid = (uintptr_t)curthread;
 1151         op = (flags & LK_TYPE_MASK);
 1152         iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
 1153         ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
 1154         itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
 1155 
 1156         lwa.iwmesg = iwmesg;
 1157         lwa.ipri = ipri;
 1158         lwa.itimo = itimo;
 1159 
 1160         MPASS((flags & ~LK_TOTAL_MASK) == 0);
 1161         KASSERT((op & (op - 1)) == 0,
 1162             ("%s: Invalid requested operation @ %s:%d", __func__, file, line));
 1163         KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
 1164             (op != LK_DOWNGRADE && op != LK_RELEASE),
 1165             ("%s: Invalid flags in regard of the operation desired @ %s:%d",
 1166             __func__, file, line));
 1167         KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
 1168             ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
 1169             __func__, file, line));
 1170         KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
 1171             ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
 1172             lk->lock_object.lo_name, file, line));
 1173 
 1174         class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
 1175 
 1176         if (lk->lock_object.lo_flags & LK_NOSHARE) {
 1177                 switch (op) {
 1178                 case LK_SHARED:
 1179                         op = LK_EXCLUSIVE;
 1180                         break;
 1181                 case LK_UPGRADE:
 1182                 case LK_TRYUPGRADE:
 1183                 case LK_DOWNGRADE:
 1184                         _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
 1185                             file, line);
 1186                         if (flags & LK_INTERLOCK)
 1187                                 class->lc_unlock(ilk);
 1188                         return (0);
 1189                 }
 1190         }
 1191 
 1192         wakeup_swapper = 0;
 1193         switch (op) {
 1194         case LK_SHARED:
 1195                 return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
 1196                 break;
 1197         case LK_UPGRADE:
 1198         case LK_TRYUPGRADE:
 1199                 return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
 1200                 break;
 1201         case LK_EXCLUSIVE:
 1202                 return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
 1203                 break;
 1204         case LK_DOWNGRADE:
 1205                 _lockmgr_assert(lk, KA_XLOCKED, file, line);
 1206                 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
 1207                 WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
 1208 
 1209                 /*
 1210                  * Panic if the lock is recursed.
 1211                  */
 1212                 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
 1213                         if (flags & LK_INTERLOCK)
 1214                                 class->lc_unlock(ilk);
 1215                         panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
 1216                             __func__, iwmesg, file, line);
 1217                 }
 1218                 TD_SLOCKS_INC(curthread);
 1219 
 1220                 /*
 1221                  * In order to preserve waiters flags, just spin.
 1222                  */
 1223                 for (;;) {
 1224                         x = lk->lk_lock;
 1225                         MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 1226                         x &= LK_ALL_WAITERS;
 1227                         if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
 1228                             LK_SHARERS_LOCK(1) | x))
 1229                                 break;
 1230                         cpu_spinwait();
 1231                 }
 1232                 break;
 1233         case LK_RELEASE:
 1234                 _lockmgr_assert(lk, KA_LOCKED, file, line);
 1235                 x = lk->lk_lock;
 1236 
 1237                 if (__predict_true(x & LK_SHARE) != 0) {
 1238                         return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
 1239                 } else {
 1240                         return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
 1241                 }
 1242                 break;
 1243         case LK_DRAIN:
 1244                 if (LK_CAN_WITNESS(flags))
 1245                         WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
 1246                             LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
 1247                             ilk : NULL);
 1248 
 1249                 /*
 1250                  * Trying to drain a lock we already own will result in a
 1251                  * deadlock.
 1252                  */
 1253                 if (lockmgr_xlocked(lk)) {
 1254                         if (flags & LK_INTERLOCK)
 1255                                 class->lc_unlock(ilk);
 1256                         panic("%s: draining %s with the lock held @ %s:%d\n",
 1257                             __func__, iwmesg, file, line);
 1258                 }
 1259 
 1260                 for (;;) {
 1261                         if (lk->lk_lock == LK_UNLOCKED &&
 1262                             atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
 1263                                 break;
 1264 
 1265 #ifdef HWPMC_HOOKS
 1266                         PMC_SOFT_CALL( , , lock, failed);
 1267 #endif
 1268                         lock_profile_obtain_lock_failed(&lk->lock_object,
 1269                             &contested, &waittime);
 1270 
 1271                         /*
 1272                          * If the lock is expected to not sleep just give up
 1273                          * and return.
 1274                          */
 1275                         if (LK_TRYOP(flags)) {
 1276                                 LOCK_LOG2(lk, "%s: %p fails the try operation",
 1277                                     __func__, lk);
 1278                                 error = EBUSY;
 1279                                 break;
 1280                         }
 1281 
 1282                         /*
 1283                          * Acquire the sleepqueue chain lock because we
 1284                          * probabilly will need to manipulate waiters flags.
 1285                          */
 1286                         sleepq_lock(&lk->lock_object);
 1287                         x = lk->lk_lock;
 1288 
 1289                         /*
 1290                          * if the lock has been released while we spun on
 1291                          * the sleepqueue chain lock just try again.
 1292                          */
 1293                         if (x == LK_UNLOCKED) {
 1294                                 sleepq_release(&lk->lock_object);
 1295                                 continue;
 1296                         }
 1297 
 1298                         v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 1299                         if ((x & ~v) == LK_UNLOCKED) {
 1300                                 v = (x & ~LK_EXCLUSIVE_SPINNERS);
 1301 
 1302                                 /*
 1303                                  * If interruptible sleeps left the exclusive
 1304                                  * queue empty avoid a starvation for the
 1305                                  * threads sleeping on the shared queue by
 1306                                  * giving them precedence and cleaning up the
 1307                                  * exclusive waiters bit anyway.
 1308                                  * Please note that lk_exslpfail count may be
 1309                                  * lying about the real number of waiters with
 1310                                  * the LK_SLEEPFAIL flag on because they may
 1311                                  * be used in conjunction with interruptible
 1312                                  * sleeps so lk_exslpfail might be considered
 1313                                  * an 'upper limit' bound, including the edge
 1314                                  * cases.
 1315                                  */
 1316                                 if (v & LK_EXCLUSIVE_WAITERS) {
 1317                                         queue = SQ_EXCLUSIVE_QUEUE;
 1318                                         v &= ~LK_EXCLUSIVE_WAITERS;
 1319                                 } else {
 1320 
 1321                                         /*
 1322                                          * Exclusive waiters sleeping with
 1323                                          * LK_SLEEPFAIL on and using
 1324                                          * interruptible sleeps/timeout may
 1325                                          * have left spourious lk_exslpfail
 1326                                          * counts on, so clean it up anyway.
 1327                                          */
 1328                                         MPASS(v & LK_SHARED_WAITERS);
 1329                                         lk->lk_exslpfail = 0;
 1330                                         queue = SQ_SHARED_QUEUE;
 1331                                         v &= ~LK_SHARED_WAITERS;
 1332                                 }
 1333                                 if (queue == SQ_EXCLUSIVE_QUEUE) {
 1334                                         realexslp =
 1335                                             sleepq_sleepcnt(&lk->lock_object,
 1336                                             SQ_EXCLUSIVE_QUEUE);
 1337                                         if (lk->lk_exslpfail >= realexslp) {
 1338                                                 lk->lk_exslpfail = 0;
 1339                                                 queue = SQ_SHARED_QUEUE;
 1340                                                 v &= ~LK_SHARED_WAITERS;
 1341                                                 if (realexslp != 0) {
 1342                                                         LOCK_LOG2(lk,
 1343                                         "%s: %p has only LK_SLEEPFAIL sleepers",
 1344                                                             __func__, lk);
 1345                                                         LOCK_LOG2(lk,
 1346                         "%s: %p waking up threads on the exclusive queue",
 1347                                                             __func__, lk);
 1348                                                         wakeup_swapper =
 1349                                                             sleepq_broadcast(
 1350                                                             &lk->lock_object,
 1351                                                             SLEEPQ_LK, 0,
 1352                                                             SQ_EXCLUSIVE_QUEUE);
 1353                                                 }
 1354                                         } else
 1355                                                 lk->lk_exslpfail = 0;
 1356                                 }
 1357                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
 1358                                         sleepq_release(&lk->lock_object);
 1359                                         continue;
 1360                                 }
 1361                                 LOCK_LOG3(lk,
 1362                                 "%s: %p waking up all threads on the %s queue",
 1363                                     __func__, lk, queue == SQ_SHARED_QUEUE ?
 1364                                     "shared" : "exclusive");
 1365                                 wakeup_swapper |= sleepq_broadcast(
 1366                                     &lk->lock_object, SLEEPQ_LK, 0, queue);
 1367 
 1368                                 /*
 1369                                  * If shared waiters have been woken up we need
 1370                                  * to wait for one of them to acquire the lock
 1371                                  * before to set the exclusive waiters in
 1372                                  * order to avoid a deadlock.
 1373                                  */
 1374                                 if (queue == SQ_SHARED_QUEUE) {
 1375                                         for (v = lk->lk_lock;
 1376                                             (v & LK_SHARE) && !LK_SHARERS(v);
 1377                                             v = lk->lk_lock)
 1378                                                 cpu_spinwait();
 1379                                 }
 1380                         }
 1381 
 1382                         /*
 1383                          * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
 1384                          * fail, loop back and retry.
 1385                          */
 1386                         if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
 1387                                 if (!atomic_cmpset_ptr(&lk->lk_lock, x,
 1388                                     x | LK_EXCLUSIVE_WAITERS)) {
 1389                                         sleepq_release(&lk->lock_object);
 1390                                         continue;
 1391                                 }
 1392                                 LOCK_LOG2(lk, "%s: %p set drain waiters flag",
 1393                                     __func__, lk);
 1394                         }
 1395 
 1396                         /*
 1397                          * As far as we have been unable to acquire the
 1398                          * exclusive lock and the exclusive waiters flag
 1399                          * is set, we will sleep.
 1400                          */
 1401                         if (flags & LK_INTERLOCK) {
 1402                                 class->lc_unlock(ilk);
 1403                                 flags &= ~LK_INTERLOCK;
 1404                         }
 1405                         GIANT_SAVE();
 1406                         sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
 1407                             SQ_EXCLUSIVE_QUEUE);
 1408                         sleepq_wait(&lk->lock_object, ipri & PRIMASK);
 1409                         GIANT_RESTORE();
 1410                         LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
 1411                             __func__, lk);
 1412                 }
 1413 
 1414                 if (error == 0) {
 1415                         lock_profile_obtain_lock_success(&lk->lock_object,
 1416                             contested, waittime, file, line);
 1417                         LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
 1418                             lk->lk_recurse, file, line);
 1419                         WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
 1420                             LK_TRYWIT(flags), file, line);
 1421                         TD_LOCKS_INC(curthread);
 1422                         STACK_SAVE(lk);
 1423                 }
 1424                 break;
 1425         default:
 1426                 if (flags & LK_INTERLOCK)
 1427                         class->lc_unlock(ilk);
 1428                 panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
 1429         }
 1430 
 1431         if (flags & LK_INTERLOCK)
 1432                 class->lc_unlock(ilk);
 1433         if (wakeup_swapper)
 1434                 kick_proc0();
 1435 
 1436         return (error);
 1437 }
 1438 
 1439 void
 1440 _lockmgr_disown(struct lock *lk, const char *file, int line)
 1441 {
 1442         uintptr_t tid, x;
 1443 
 1444         if (SCHEDULER_STOPPED())
 1445                 return;
 1446 
 1447         tid = (uintptr_t)curthread;
 1448         _lockmgr_assert(lk, KA_XLOCKED, file, line);
 1449 
 1450         /*
 1451          * Panic if the lock is recursed.
 1452          */
 1453         if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
 1454                 panic("%s: disown a recursed lockmgr @ %s:%d\n",
 1455                     __func__,  file, line);
 1456 
 1457         /*
 1458          * If the owner is already LK_KERNPROC just skip the whole operation.
 1459          */
 1460         if (LK_HOLDER(lk->lk_lock) != tid)
 1461                 return;
 1462         lock_profile_release_lock(&lk->lock_object);
 1463         LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
 1464         WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
 1465         TD_LOCKS_DEC(curthread);
 1466         STACK_SAVE(lk);
 1467 
 1468         /*
 1469          * In order to preserve waiters flags, just spin.
 1470          */
 1471         for (;;) {
 1472                 x = lk->lk_lock;
 1473                 MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
 1474                 x &= LK_ALL_WAITERS;
 1475                 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
 1476                     LK_KERNPROC | x))
 1477                         return;
 1478                 cpu_spinwait();
 1479         }
 1480 }
 1481 
 1482 void
 1483 lockmgr_printinfo(const struct lock *lk)
 1484 {
 1485         struct thread *td;
 1486         uintptr_t x;
 1487 
 1488         if (lk->lk_lock == LK_UNLOCKED)
 1489                 printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
 1490         else if (lk->lk_lock & LK_SHARE)
 1491                 printf("lock type %s: SHARED (count %ju)\n",
 1492                     lk->lock_object.lo_name,
 1493                     (uintmax_t)LK_SHARERS(lk->lk_lock));
 1494         else {
 1495                 td = lockmgr_xholder(lk);
 1496                 if (td == (struct thread *)LK_KERNPROC)
 1497                         printf("lock type %s: EXCL by KERNPROC\n",
 1498                             lk->lock_object.lo_name);
 1499                 else
 1500                         printf("lock type %s: EXCL by thread %p "
 1501                             "(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
 1502                             td, td->td_proc->p_pid, td->td_proc->p_comm,
 1503                             td->td_tid);
 1504         }
 1505 
 1506         x = lk->lk_lock;
 1507         if (x & LK_EXCLUSIVE_WAITERS)
 1508                 printf(" with exclusive waiters pending\n");
 1509         if (x & LK_SHARED_WAITERS)
 1510                 printf(" with shared waiters pending\n");
 1511         if (x & LK_EXCLUSIVE_SPINNERS)
 1512                 printf(" with exclusive spinners pending\n");
 1513 
 1514         STACK_PRINT(lk);
 1515 }
 1516 
 1517 int
 1518 lockstatus(const struct lock *lk)
 1519 {
 1520         uintptr_t v, x;
 1521         int ret;
 1522 
 1523         ret = LK_SHARED;
 1524         x = lk->lk_lock;
 1525         v = LK_HOLDER(x);
 1526 
 1527         if ((x & LK_SHARE) == 0) {
 1528                 if (v == (uintptr_t)curthread || v == LK_KERNPROC)
 1529                         ret = LK_EXCLUSIVE;
 1530                 else
 1531                         ret = LK_EXCLOTHER;
 1532         } else if (x == LK_UNLOCKED)
 1533                 ret = 0;
 1534 
 1535         return (ret);
 1536 }
 1537 
 1538 #ifdef INVARIANT_SUPPORT
 1539 
 1540 FEATURE(invariant_support,
 1541     "Support for modules compiled with INVARIANTS option");
 1542 
 1543 #ifndef INVARIANTS
 1544 #undef  _lockmgr_assert
 1545 #endif
 1546 
 1547 void
 1548 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
 1549 {
 1550         int slocked = 0;
 1551 
 1552         if (panicstr != NULL)
 1553                 return;
 1554         switch (what) {
 1555         case KA_SLOCKED:
 1556         case KA_SLOCKED | KA_NOTRECURSED:
 1557         case KA_SLOCKED | KA_RECURSED:
 1558                 slocked = 1;
 1559         case KA_LOCKED:
 1560         case KA_LOCKED | KA_NOTRECURSED:
 1561         case KA_LOCKED | KA_RECURSED:
 1562 #ifdef WITNESS
 1563 
 1564                 /*
 1565                  * We cannot trust WITNESS if the lock is held in exclusive
 1566                  * mode and a call to lockmgr_disown() happened.
 1567                  * Workaround this skipping the check if the lock is held in
 1568                  * exclusive mode even for the KA_LOCKED case.
 1569                  */
 1570                 if (slocked || (lk->lk_lock & LK_SHARE)) {
 1571                         witness_assert(&lk->lock_object, what, file, line);
 1572                         break;
 1573                 }
 1574 #endif
 1575                 if (lk->lk_lock == LK_UNLOCKED ||
 1576                     ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
 1577                     (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
 1578                         panic("Lock %s not %slocked @ %s:%d\n",
 1579                             lk->lock_object.lo_name, slocked ? "share" : "",
 1580                             file, line);
 1581 
 1582                 if ((lk->lk_lock & LK_SHARE) == 0) {
 1583                         if (lockmgr_recursed(lk)) {
 1584                                 if (what & KA_NOTRECURSED)
 1585                                         panic("Lock %s recursed @ %s:%d\n",
 1586                                             lk->lock_object.lo_name, file,
 1587                                             line);
 1588                         } else if (what & KA_RECURSED)
 1589                                 panic("Lock %s not recursed @ %s:%d\n",
 1590                                     lk->lock_object.lo_name, file, line);
 1591                 }
 1592                 break;
 1593         case KA_XLOCKED:
 1594         case KA_XLOCKED | KA_NOTRECURSED:
 1595         case KA_XLOCKED | KA_RECURSED:
 1596                 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
 1597                         panic("Lock %s not exclusively locked @ %s:%d\n",
 1598                             lk->lock_object.lo_name, file, line);
 1599                 if (lockmgr_recursed(lk)) {
 1600                         if (what & KA_NOTRECURSED)
 1601                                 panic("Lock %s recursed @ %s:%d\n",
 1602                                     lk->lock_object.lo_name, file, line);
 1603                 } else if (what & KA_RECURSED)
 1604                         panic("Lock %s not recursed @ %s:%d\n",
 1605                             lk->lock_object.lo_name, file, line);
 1606                 break;
 1607         case KA_UNLOCKED:
 1608                 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
 1609                         panic("Lock %s exclusively locked @ %s:%d\n",
 1610                             lk->lock_object.lo_name, file, line);
 1611                 break;
 1612         default:
 1613                 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
 1614                     line);
 1615         }
 1616 }
 1617 #endif
 1618 
 1619 #ifdef DDB
 1620 int
 1621 lockmgr_chain(struct thread *td, struct thread **ownerp)
 1622 {
 1623         struct lock *lk;
 1624 
 1625         lk = td->td_wchan;
 1626 
 1627         if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
 1628                 return (0);
 1629         db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
 1630         if (lk->lk_lock & LK_SHARE)
 1631                 db_printf("SHARED (count %ju)\n",
 1632                     (uintmax_t)LK_SHARERS(lk->lk_lock));
 1633         else
 1634                 db_printf("EXCL\n");
 1635         *ownerp = lockmgr_xholder(lk);
 1636 
 1637         return (1);
 1638 }
 1639 
 1640 static void
 1641 db_show_lockmgr(const struct lock_object *lock)
 1642 {
 1643         struct thread *td;
 1644         const struct lock *lk;
 1645 
 1646         lk = (const struct lock *)lock;
 1647 
 1648         db_printf(" state: ");
 1649         if (lk->lk_lock == LK_UNLOCKED)
 1650                 db_printf("UNLOCKED\n");
 1651         else if (lk->lk_lock & LK_SHARE)
 1652                 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
 1653         else {
 1654                 td = lockmgr_xholder(lk);
 1655                 if (td == (struct thread *)LK_KERNPROC)
 1656                         db_printf("XLOCK: LK_KERNPROC\n");
 1657                 else
 1658                         db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
 1659                             td->td_tid, td->td_proc->p_pid,
 1660                             td->td_proc->p_comm);
 1661                 if (lockmgr_recursed(lk))
 1662                         db_printf(" recursed: %d\n", lk->lk_recurse);
 1663         }
 1664         db_printf(" waiters: ");
 1665         switch (lk->lk_lock & LK_ALL_WAITERS) {
 1666         case LK_SHARED_WAITERS:
 1667                 db_printf("shared\n");
 1668                 break;
 1669         case LK_EXCLUSIVE_WAITERS:
 1670                 db_printf("exclusive\n");
 1671                 break;
 1672         case LK_ALL_WAITERS:
 1673                 db_printf("shared and exclusive\n");
 1674                 break;
 1675         default:
 1676                 db_printf("none\n");
 1677         }
 1678         db_printf(" spinners: ");
 1679         if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
 1680                 db_printf("exclusive\n");
 1681         else
 1682                 db_printf("none\n");
 1683 }
 1684 #endif

Cache object: bbca2ee5e84f9934b0f328e95f008e29


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.