The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mutex.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  * 3. Berkeley Software Design Inc's name may not be used to endorse or
   13  *    promote products derived from this software without specific prior
   14  *    written permission.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  *      from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
   29  *      and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
   30  */
   31 
   32 /*
   33  * Machine independent bits of mutex implementation.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD: releng/8.3/sys/kern/kern_mutex.c 208261 2010-05-18 10:24:23Z attilio $");
   38 
   39 #include "opt_adaptive_mutexes.h"
   40 #include "opt_ddb.h"
   41 #include "opt_global.h"
   42 #include "opt_kdtrace.h"
   43 #include "opt_sched.h"
   44 
   45 #include <sys/param.h>
   46 #include <sys/systm.h>
   47 #include <sys/bus.h>
   48 #include <sys/conf.h>
   49 #include <sys/kdb.h>
   50 #include <sys/kernel.h>
   51 #include <sys/ktr.h>
   52 #include <sys/lock.h>
   53 #include <sys/malloc.h>
   54 #include <sys/mutex.h>
   55 #include <sys/proc.h>
   56 #include <sys/resourcevar.h>
   57 #include <sys/sched.h>
   58 #include <sys/sbuf.h>
   59 #include <sys/sysctl.h>
   60 #include <sys/turnstile.h>
   61 #include <sys/vmmeter.h>
   62 #include <sys/lock_profile.h>
   63 
   64 #include <machine/atomic.h>
   65 #include <machine/bus.h>
   66 #include <machine/cpu.h>
   67 
   68 #include <ddb/ddb.h>
   69 
   70 #include <fs/devfs/devfs_int.h>
   71 
   72 #include <vm/vm.h>
   73 #include <vm/vm_extern.h>
   74 
   75 #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
   76 #define ADAPTIVE_MUTEXES
   77 #endif
   78 
   79 /*
   80  * Internal utility macros.
   81  */
   82 #define mtx_unowned(m)  ((m)->mtx_lock == MTX_UNOWNED)
   83 
   84 #define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
   85 
   86 #define mtx_owner(m)    ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
   87 
   88 static void     assert_mtx(struct lock_object *lock, int what);
   89 #ifdef DDB
   90 static void     db_show_mtx(struct lock_object *lock);
   91 #endif
   92 static void     lock_mtx(struct lock_object *lock, int how);
   93 static void     lock_spin(struct lock_object *lock, int how);
   94 #ifdef KDTRACE_HOOKS
   95 static int      owner_mtx(struct lock_object *lock, struct thread **owner);
   96 #endif
   97 static int      unlock_mtx(struct lock_object *lock);
   98 static int      unlock_spin(struct lock_object *lock);
   99 
  100 /*
  101  * Lock classes for sleep and spin mutexes.
  102  */
  103 struct lock_class lock_class_mtx_sleep = {
  104         .lc_name = "sleep mutex",
  105         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
  106         .lc_assert = assert_mtx,
  107 #ifdef DDB
  108         .lc_ddb_show = db_show_mtx,
  109 #endif
  110         .lc_lock = lock_mtx,
  111         .lc_unlock = unlock_mtx,
  112 #ifdef KDTRACE_HOOKS
  113         .lc_owner = owner_mtx,
  114 #endif
  115 };
  116 struct lock_class lock_class_mtx_spin = {
  117         .lc_name = "spin mutex",
  118         .lc_flags = LC_SPINLOCK | LC_RECURSABLE,
  119         .lc_assert = assert_mtx,
  120 #ifdef DDB
  121         .lc_ddb_show = db_show_mtx,
  122 #endif
  123         .lc_lock = lock_spin,
  124         .lc_unlock = unlock_spin,
  125 #ifdef KDTRACE_HOOKS
  126         .lc_owner = owner_mtx,
  127 #endif
  128 };
  129 
  130 /*
  131  * System-wide mutexes
  132  */
  133 struct mtx blocked_lock;
  134 struct mtx Giant;
  135 
  136 void
  137 assert_mtx(struct lock_object *lock, int what)
  138 {
  139 
  140         mtx_assert((struct mtx *)lock, what);
  141 }
  142 
  143 void
  144 lock_mtx(struct lock_object *lock, int how)
  145 {
  146 
  147         mtx_lock((struct mtx *)lock);
  148 }
  149 
  150 void
  151 lock_spin(struct lock_object *lock, int how)
  152 {
  153 
  154         panic("spin locks can only use msleep_spin");
  155 }
  156 
  157 int
  158 unlock_mtx(struct lock_object *lock)
  159 {
  160         struct mtx *m;
  161 
  162         m = (struct mtx *)lock;
  163         mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
  164         mtx_unlock(m);
  165         return (0);
  166 }
  167 
  168 int
  169 unlock_spin(struct lock_object *lock)
  170 {
  171 
  172         panic("spin locks can only use msleep_spin");
  173 }
  174 
  175 #ifdef KDTRACE_HOOKS
  176 int
  177 owner_mtx(struct lock_object *lock, struct thread **owner)
  178 {
  179         struct mtx *m = (struct mtx *)lock;
  180 
  181         *owner = mtx_owner(m);
  182         return (mtx_unowned(m) == 0);
  183 }
  184 #endif
  185 
  186 /*
  187  * Function versions of the inlined __mtx_* macros.  These are used by
  188  * modules and can also be called from assembly language if needed.
  189  */
  190 void
  191 _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
  192 {
  193 
  194         MPASS(curthread != NULL);
  195         KASSERT(m->mtx_lock != MTX_DESTROYED,
  196             ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
  197         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  198             ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  199             file, line));
  200         WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
  201             file, line, NULL);
  202 
  203         _get_sleep_lock(m, curthread, opts, file, line);
  204         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  205             line);
  206         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  207         curthread->td_locks++;
  208 }
  209 
  210 void
  211 _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
  212 {
  213         MPASS(curthread != NULL);
  214         KASSERT(m->mtx_lock != MTX_DESTROYED,
  215             ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
  216         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  217             ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  218             file, line));
  219         curthread->td_locks--;
  220         WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  221         LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
  222             line);
  223         mtx_assert(m, MA_OWNED);
  224 
  225         if (m->mtx_recurse == 0)
  226                 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m);
  227         _rel_sleep_lock(m, curthread, opts, file, line);
  228 }
  229 
  230 void
  231 _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
  232 {
  233 
  234         MPASS(curthread != NULL);
  235         KASSERT(m->mtx_lock != MTX_DESTROYED,
  236             ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
  237         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  238             ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
  239             m->lock_object.lo_name, file, line));
  240         if (mtx_owned(m))
  241                 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  242             ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
  243                     m->lock_object.lo_name, file, line));
  244         WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
  245             file, line, NULL);
  246         _get_spin_lock(m, curthread, opts, file, line);
  247         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  248             line);
  249         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  250 }
  251 
  252 void
  253 _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
  254 {
  255 
  256         MPASS(curthread != NULL);
  257         KASSERT(m->mtx_lock != MTX_DESTROYED,
  258             ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
  259         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  260             ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
  261             m->lock_object.lo_name, file, line));
  262         WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  263         LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
  264             line);
  265         mtx_assert(m, MA_OWNED);
  266 
  267         _rel_spin_lock(m);
  268 }
  269 
  270 /*
  271  * The important part of mtx_trylock{,_flags}()
  272  * Tries to acquire lock `m.'  If this function is called on a mutex that
  273  * is already owned, it will recursively acquire the lock.
  274  */
  275 int
  276 _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
  277 {
  278 #ifdef LOCK_PROFILING
  279         uint64_t waittime = 0;
  280         int contested = 0;
  281 #endif
  282         int rval;
  283 
  284         MPASS(curthread != NULL);
  285         KASSERT(m->mtx_lock != MTX_DESTROYED,
  286             ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
  287         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  288             ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  289             file, line));
  290 
  291         if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  292                 m->mtx_recurse++;
  293                 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
  294                 rval = 1;
  295         } else
  296                 rval = _obtain_lock(m, (uintptr_t)curthread);
  297 
  298         LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
  299         if (rval) {
  300                 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
  301                     file, line);
  302                 curthread->td_locks++;
  303                 if (m->mtx_recurse == 0)
  304                         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE,
  305                             m, contested, waittime, file, line);
  306 
  307         }
  308 
  309         return (rval);
  310 }
  311 
  312 /*
  313  * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
  314  *
  315  * We call this if the lock is either contested (i.e. we need to go to
  316  * sleep waiting for it), or if we need to recurse on it.
  317  */
  318 void
  319 _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
  320     int line)
  321 {
  322         struct turnstile *ts;
  323         uintptr_t v;
  324 #ifdef ADAPTIVE_MUTEXES
  325         volatile struct thread *owner;
  326 #endif
  327 #ifdef KTR
  328         int cont_logged = 0;
  329 #endif
  330 #ifdef LOCK_PROFILING
  331         int contested = 0;
  332         uint64_t waittime = 0;
  333 #endif
  334 #ifdef KDTRACE_HOOKS
  335         uint64_t spin_cnt = 0;
  336         uint64_t sleep_cnt = 0;
  337         int64_t sleep_time = 0;
  338 #endif
  339 
  340         if (mtx_owned(m)) {
  341                 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  342             ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
  343                     m->lock_object.lo_name, file, line));
  344                 m->mtx_recurse++;
  345                 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
  346                 if (LOCK_LOG_TEST(&m->lock_object, opts))
  347                         CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
  348                 return;
  349         }
  350 
  351         lock_profile_obtain_lock_failed(&m->lock_object,
  352                     &contested, &waittime);
  353         if (LOCK_LOG_TEST(&m->lock_object, opts))
  354                 CTR4(KTR_LOCK,
  355                     "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
  356                     m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
  357 
  358         while (!_obtain_lock(m, tid)) {
  359 #ifdef KDTRACE_HOOKS
  360                 spin_cnt++;
  361 #endif
  362 #ifdef ADAPTIVE_MUTEXES
  363                 /*
  364                  * If the owner is running on another CPU, spin until the
  365                  * owner stops running or the state of the lock changes.
  366                  */
  367                 v = m->mtx_lock;
  368                 if (v != MTX_UNOWNED) {
  369                         owner = (struct thread *)(v & ~MTX_FLAGMASK);
  370                         if (TD_IS_RUNNING(owner)) {
  371                                 if (LOCK_LOG_TEST(&m->lock_object, 0))
  372                                         CTR3(KTR_LOCK,
  373                                             "%s: spinning on %p held by %p",
  374                                             __func__, m, owner);
  375                                 while (mtx_owner(m) == owner &&
  376                                     TD_IS_RUNNING(owner)) {
  377                                         cpu_spinwait();
  378 #ifdef KDTRACE_HOOKS
  379                                         spin_cnt++;
  380 #endif
  381                                 }
  382                                 continue;
  383                         }
  384                 }
  385 #endif
  386 
  387                 ts = turnstile_trywait(&m->lock_object);
  388                 v = m->mtx_lock;
  389 
  390                 /*
  391                  * Check if the lock has been released while spinning for
  392                  * the turnstile chain lock.
  393                  */
  394                 if (v == MTX_UNOWNED) {
  395                         turnstile_cancel(ts);
  396                         continue;
  397                 }
  398 
  399 #ifdef ADAPTIVE_MUTEXES
  400                 /*
  401                  * The current lock owner might have started executing
  402                  * on another CPU (or the lock could have changed
  403                  * owners) while we were waiting on the turnstile
  404                  * chain lock.  If so, drop the turnstile lock and try
  405                  * again.
  406                  */
  407                 owner = (struct thread *)(v & ~MTX_FLAGMASK);
  408                 if (TD_IS_RUNNING(owner)) {
  409                         turnstile_cancel(ts);
  410                         continue;
  411                 }
  412 #endif
  413 
  414                 /*
  415                  * If the mutex isn't already contested and a failure occurs
  416                  * setting the contested bit, the mutex was either released
  417                  * or the state of the MTX_RECURSED bit changed.
  418                  */
  419                 if ((v & MTX_CONTESTED) == 0 &&
  420                     !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
  421                         turnstile_cancel(ts);
  422                         continue;
  423                 }
  424 
  425                 /*
  426                  * We definitely must sleep for this lock.
  427                  */
  428                 mtx_assert(m, MA_NOTOWNED);
  429 
  430 #ifdef KTR
  431                 if (!cont_logged) {
  432                         CTR6(KTR_CONTENTION,
  433                             "contention: %p at %s:%d wants %s, taken by %s:%d",
  434                             (void *)tid, file, line, m->lock_object.lo_name,
  435                             WITNESS_FILE(&m->lock_object),
  436                             WITNESS_LINE(&m->lock_object));
  437                         cont_logged = 1;
  438                 }
  439 #endif
  440 
  441                 /*
  442                  * Block on the turnstile.
  443                  */
  444 #ifdef KDTRACE_HOOKS
  445                 sleep_time -= lockstat_nsecs();
  446 #endif
  447                 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
  448 #ifdef KDTRACE_HOOKS
  449                 sleep_time += lockstat_nsecs();
  450                 sleep_cnt++;
  451 #endif
  452         }
  453 #ifdef KTR
  454         if (cont_logged) {
  455                 CTR4(KTR_CONTENTION,
  456                     "contention end: %s acquired by %p at %s:%d",
  457                     m->lock_object.lo_name, (void *)tid, file, line);
  458         }
  459 #endif
  460         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested,
  461             waittime, file, line);
  462 #ifdef KDTRACE_HOOKS
  463         if (sleep_time)
  464                 LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time);
  465 
  466         /*
  467          * Only record the loops spinning and not sleeping. 
  468          */
  469         if (spin_cnt > sleep_cnt)
  470                 LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt));
  471 #endif
  472 }
  473 
  474 static void
  475 _mtx_lock_spin_failed(struct mtx *m)
  476 {
  477         struct thread *td;
  478 
  479         td = mtx_owner(m);
  480 
  481         /* If the mutex is unlocked, try again. */
  482         if (td == NULL)
  483                 return;
  484 
  485         printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
  486             m, m->lock_object.lo_name, td, td->td_tid);
  487 #ifdef WITNESS
  488         witness_display_spinlock(&m->lock_object, td, printf);
  489 #endif
  490         panic("spin lock held too long");
  491 }
  492 
  493 #ifdef SMP
  494 /*
  495  * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
  496  *
  497  * This is only called if we need to actually spin for the lock. Recursion
  498  * is handled inline.
  499  */
  500 void
  501 _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
  502     int line)
  503 {
  504         int i = 0;
  505 #ifdef LOCK_PROFILING
  506         int contested = 0;
  507         uint64_t waittime = 0;
  508 #endif
  509 
  510         if (LOCK_LOG_TEST(&m->lock_object, opts))
  511                 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
  512 
  513         lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
  514         while (!_obtain_lock(m, tid)) {
  515 
  516                 /* Give interrupts a chance while we spin. */
  517                 spinlock_exit();
  518                 while (m->mtx_lock != MTX_UNOWNED) {
  519                         if (i++ < 10000000) {
  520                                 cpu_spinwait();
  521                                 continue;
  522                         }
  523                         if (i < 60000000 || kdb_active || panicstr != NULL)
  524                                 DELAY(1);
  525                         else
  526                                 _mtx_lock_spin_failed(m);
  527                         cpu_spinwait();
  528                 }
  529                 spinlock_enter();
  530         }
  531 
  532         if (LOCK_LOG_TEST(&m->lock_object, opts))
  533                 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
  534 
  535         LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m,
  536             contested, waittime, (file), (line));
  537         LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i);
  538 }
  539 #endif /* SMP */
  540 
  541 void
  542 _thread_lock_flags(struct thread *td, int opts, const char *file, int line)
  543 {
  544         struct mtx *m;
  545         uintptr_t tid;
  546         int i;
  547 #ifdef LOCK_PROFILING
  548         int contested = 0;
  549         uint64_t waittime = 0;
  550 #endif
  551 #ifdef KDTRACE_HOOKS
  552         uint64_t spin_cnt = 0;
  553 #endif
  554 
  555         i = 0;
  556         tid = (uintptr_t)curthread;
  557         for (;;) {
  558 retry:
  559                 spinlock_enter();
  560                 m = td->td_lock;
  561                 KASSERT(m->mtx_lock != MTX_DESTROYED,
  562                     ("thread_lock() of destroyed mutex @ %s:%d", file, line));
  563                 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  564                     ("thread_lock() of sleep mutex %s @ %s:%d",
  565                     m->lock_object.lo_name, file, line));
  566                 if (mtx_owned(m))
  567                         KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  568             ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
  569                             m->lock_object.lo_name, file, line));
  570                 WITNESS_CHECKORDER(&m->lock_object,
  571                     opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
  572                 while (!_obtain_lock(m, tid)) {
  573 #ifdef KDTRACE_HOOKS
  574                         spin_cnt++;
  575 #endif
  576                         if (m->mtx_lock == tid) {
  577                                 m->mtx_recurse++;
  578                                 break;
  579                         }
  580                         lock_profile_obtain_lock_failed(&m->lock_object,
  581                             &contested, &waittime);
  582                         /* Give interrupts a chance while we spin. */
  583                         spinlock_exit();
  584                         while (m->mtx_lock != MTX_UNOWNED) {
  585                                 if (i++ < 10000000)
  586                                         cpu_spinwait();
  587                                 else if (i < 60000000 ||
  588                                     kdb_active || panicstr != NULL)
  589                                         DELAY(1);
  590                                 else
  591                                         _mtx_lock_spin_failed(m);
  592                                 cpu_spinwait();
  593                                 if (m != td->td_lock)
  594                                         goto retry;
  595                         }
  596                         spinlock_enter();
  597                 }
  598                 if (m == td->td_lock)
  599                         break;
  600                 _rel_spin_lock(m);      /* does spinlock_exit() */
  601 #ifdef KDTRACE_HOOKS
  602                 spin_cnt++;
  603 #endif
  604         }
  605         if (m->mtx_recurse == 0)
  606                 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE,
  607                     m, contested, waittime, (file), (line));
  608         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  609             line);
  610         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  611         LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt);
  612 }
  613 
  614 struct mtx *
  615 thread_lock_block(struct thread *td)
  616 {
  617         struct mtx *lock;
  618 
  619         THREAD_LOCK_ASSERT(td, MA_OWNED);
  620         lock = td->td_lock;
  621         td->td_lock = &blocked_lock;
  622         mtx_unlock_spin(lock);
  623 
  624         return (lock);
  625 }
  626 
  627 void
  628 thread_lock_unblock(struct thread *td, struct mtx *new)
  629 {
  630         mtx_assert(new, MA_OWNED);
  631         MPASS(td->td_lock == &blocked_lock);
  632         atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
  633 }
  634 
  635 void
  636 thread_lock_set(struct thread *td, struct mtx *new)
  637 {
  638         struct mtx *lock;
  639 
  640         mtx_assert(new, MA_OWNED);
  641         THREAD_LOCK_ASSERT(td, MA_OWNED);
  642         lock = td->td_lock;
  643         td->td_lock = new;
  644         mtx_unlock_spin(lock);
  645 }
  646 
  647 /*
  648  * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
  649  *
  650  * We are only called here if the lock is recursed or contested (i.e. we
  651  * need to wake up a blocked thread).
  652  */
  653 void
  654 _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
  655 {
  656         struct turnstile *ts;
  657 
  658         if (mtx_recursed(m)) {
  659                 if (--(m->mtx_recurse) == 0)
  660                         atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
  661                 if (LOCK_LOG_TEST(&m->lock_object, opts))
  662                         CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
  663                 return;
  664         }
  665 
  666         /*
  667          * We have to lock the chain before the turnstile so this turnstile
  668          * can be removed from the hash list if it is empty.
  669          */
  670         turnstile_chain_lock(&m->lock_object);
  671         ts = turnstile_lookup(&m->lock_object);
  672         if (LOCK_LOG_TEST(&m->lock_object, opts))
  673                 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
  674         MPASS(ts != NULL);
  675         turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
  676         _release_lock_quick(m);
  677 
  678         /*
  679          * This turnstile is now no longer associated with the mutex.  We can
  680          * unlock the chain lock so a new turnstile may take it's place.
  681          */
  682         turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  683         turnstile_chain_unlock(&m->lock_object);
  684 }
  685 
  686 /*
  687  * All the unlocking of MTX_SPIN locks is done inline.
  688  * See the _rel_spin_lock() macro for the details.
  689  */
  690 
  691 /*
  692  * The backing function for the INVARIANTS-enabled mtx_assert()
  693  */
  694 #ifdef INVARIANT_SUPPORT
  695 void
  696 _mtx_assert(struct mtx *m, int what, const char *file, int line)
  697 {
  698 
  699         if (panicstr != NULL || dumping)
  700                 return;
  701         switch (what) {
  702         case MA_OWNED:
  703         case MA_OWNED | MA_RECURSED:
  704         case MA_OWNED | MA_NOTRECURSED:
  705                 if (!mtx_owned(m))
  706                         panic("mutex %s not owned at %s:%d",
  707                             m->lock_object.lo_name, file, line);
  708                 if (mtx_recursed(m)) {
  709                         if ((what & MA_NOTRECURSED) != 0)
  710                                 panic("mutex %s recursed at %s:%d",
  711                                     m->lock_object.lo_name, file, line);
  712                 } else if ((what & MA_RECURSED) != 0) {
  713                         panic("mutex %s unrecursed at %s:%d",
  714                             m->lock_object.lo_name, file, line);
  715                 }
  716                 break;
  717         case MA_NOTOWNED:
  718                 if (mtx_owned(m))
  719                         panic("mutex %s owned at %s:%d",
  720                             m->lock_object.lo_name, file, line);
  721                 break;
  722         default:
  723                 panic("unknown mtx_assert at %s:%d", file, line);
  724         }
  725 }
  726 #endif
  727 
  728 /*
  729  * The MUTEX_DEBUG-enabled mtx_validate()
  730  *
  731  * Most of these checks have been moved off into the LO_INITIALIZED flag
  732  * maintained by the witness code.
  733  */
  734 #ifdef MUTEX_DEBUG
  735 
  736 void    mtx_validate(struct mtx *);
  737 
  738 void
  739 mtx_validate(struct mtx *m)
  740 {
  741 
  742 /*
  743  * XXX: When kernacc() does not require Giant we can reenable this check
  744  */
  745 #ifdef notyet
  746         /*
  747          * Can't call kernacc() from early init386(), especially when
  748          * initializing Giant mutex, because some stuff in kernacc()
  749          * requires Giant itself.
  750          */
  751         if (!cold)
  752                 if (!kernacc((caddr_t)m, sizeof(m),
  753                     VM_PROT_READ | VM_PROT_WRITE))
  754                         panic("Can't read and write to mutex %p", m);
  755 #endif
  756 }
  757 #endif
  758 
  759 /*
  760  * General init routine used by the MTX_SYSINIT() macro.
  761  */
  762 void
  763 mtx_sysinit(void *arg)
  764 {
  765         struct mtx_args *margs = arg;
  766 
  767         mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts);
  768 }
  769 
  770 /*
  771  * Mutex initialization routine; initialize lock `m' of type contained in
  772  * `opts' with options contained in `opts' and name `name.'  The optional
  773  * lock type `type' is used as a general lock category name for use with
  774  * witness.
  775  */
  776 void
  777 mtx_init(struct mtx *m, const char *name, const char *type, int opts)
  778 {
  779         struct lock_class *class;
  780         int flags;
  781 
  782         MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
  783                 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0);
  784         ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock,
  785             ("%s: mtx_lock not aligned for %s: %p", __func__, name,
  786             &m->mtx_lock));
  787 
  788 #ifdef MUTEX_DEBUG
  789         /* Diagnostic and error correction */
  790         mtx_validate(m);
  791 #endif
  792 
  793         /* Determine lock class and lock flags. */
  794         if (opts & MTX_SPIN)
  795                 class = &lock_class_mtx_spin;
  796         else
  797                 class = &lock_class_mtx_sleep;
  798         flags = 0;
  799         if (opts & MTX_QUIET)
  800                 flags |= LO_QUIET;
  801         if (opts & MTX_RECURSE)
  802                 flags |= LO_RECURSABLE;
  803         if ((opts & MTX_NOWITNESS) == 0)
  804                 flags |= LO_WITNESS;
  805         if (opts & MTX_DUPOK)
  806                 flags |= LO_DUPOK;
  807         if (opts & MTX_NOPROFILE)
  808                 flags |= LO_NOPROFILE;
  809 
  810         /* Initialize mutex. */
  811         m->mtx_lock = MTX_UNOWNED;
  812         m->mtx_recurse = 0;
  813 
  814         lock_init(&m->lock_object, class, name, type, flags);
  815 }
  816 
  817 /*
  818  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
  819  * passed in as a flag here because if the corresponding mtx_init() was
  820  * called with MTX_QUIET set, then it will already be set in the mutex's
  821  * flags.
  822  */
  823 void
  824 mtx_destroy(struct mtx *m)
  825 {
  826 
  827         if (!mtx_owned(m))
  828                 MPASS(mtx_unowned(m));
  829         else {
  830                 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
  831 
  832                 /* Perform the non-mtx related part of mtx_unlock_spin(). */
  833                 if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
  834                         spinlock_exit();
  835                 else
  836                         curthread->td_locks--;
  837 
  838                 lock_profile_release_lock(&m->lock_object);
  839                 /* Tell witness this isn't locked to make it happy. */
  840                 WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
  841                     __LINE__);
  842         }
  843 
  844         m->mtx_lock = MTX_DESTROYED;
  845         lock_destroy(&m->lock_object);
  846 }
  847 
  848 /*
  849  * Intialize the mutex code and system mutexes.  This is called from the MD
  850  * startup code prior to mi_startup().  The per-CPU data space needs to be
  851  * setup before this is called.
  852  */
  853 void
  854 mutex_init(void)
  855 {
  856 
  857         /* Setup turnstiles so that sleep mutexes work. */
  858         init_turnstiles();
  859 
  860         /*
  861          * Initialize mutexes.
  862          */
  863         mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
  864         mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
  865         blocked_lock.mtx_lock = 0xdeadc0de;     /* Always blocked. */
  866         mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
  867         mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE);
  868         mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
  869         mtx_lock(&Giant);
  870 }
  871 
  872 #ifdef DDB
  873 void
  874 db_show_mtx(struct lock_object *lock)
  875 {
  876         struct thread *td;
  877         struct mtx *m;
  878 
  879         m = (struct mtx *)lock;
  880 
  881         db_printf(" flags: {");
  882         if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
  883                 db_printf("SPIN");
  884         else
  885                 db_printf("DEF");
  886         if (m->lock_object.lo_flags & LO_RECURSABLE)
  887                 db_printf(", RECURSE");
  888         if (m->lock_object.lo_flags & LO_DUPOK)
  889                 db_printf(", DUPOK");
  890         db_printf("}\n");
  891         db_printf(" state: {");
  892         if (mtx_unowned(m))
  893                 db_printf("UNOWNED");
  894         else if (mtx_destroyed(m))
  895                 db_printf("DESTROYED");
  896         else {
  897                 db_printf("OWNED");
  898                 if (m->mtx_lock & MTX_CONTESTED)
  899                         db_printf(", CONTESTED");
  900                 if (m->mtx_lock & MTX_RECURSED)
  901                         db_printf(", RECURSED");
  902         }
  903         db_printf("}\n");
  904         if (!mtx_unowned(m) && !mtx_destroyed(m)) {
  905                 td = mtx_owner(m);
  906                 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
  907                     td->td_tid, td->td_proc->p_pid, td->td_name);
  908                 if (mtx_recursed(m))
  909                         db_printf(" recursed: %d\n", m->mtx_recurse);
  910         }
  911 }
  912 #endif

Cache object: db0027ba9b52a372b8a4473f9f73c110


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.