The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mutex.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  * 3. Berkeley Software Design Inc's name may not be used to endorse or
   13  *    promote products derived from this software without specific prior
   14  *    written permission.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  *      from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
   29  *      and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
   30  */
   31 
   32 /*
   33  * Machine independent bits of mutex implementation.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD$");
   38 
   39 #include "opt_adaptive_mutexes.h"
   40 #include "opt_ddb.h"
   41 #include "opt_global.h"
   42 #include "opt_sched.h"
   43 
   44 #include <sys/param.h>
   45 #include <sys/systm.h>
   46 #include <sys/bus.h>
   47 #include <sys/conf.h>
   48 #include <sys/kdb.h>
   49 #include <sys/kernel.h>
   50 #include <sys/ktr.h>
   51 #include <sys/lock.h>
   52 #include <sys/malloc.h>
   53 #include <sys/mutex.h>
   54 #include <sys/proc.h>
   55 #include <sys/resourcevar.h>
   56 #include <sys/sched.h>
   57 #include <sys/sbuf.h>
   58 #include <sys/sysctl.h>
   59 #include <sys/turnstile.h>
   60 #include <sys/vmmeter.h>
   61 #include <sys/lock_profile.h>
   62 
   63 #include <machine/atomic.h>
   64 #include <machine/bus.h>
   65 #include <machine/cpu.h>
   66 
   67 #include <ddb/ddb.h>
   68 
   69 #include <fs/devfs/devfs_int.h>
   70 
   71 #include <vm/vm.h>
   72 #include <vm/vm_extern.h>
   73 
   74 #if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
   75 #define ADAPTIVE_MUTEXES
   76 #endif
   77 
   78 /*
   79  * Internal utility macros.
   80  */
   81 #define mtx_unowned(m)  ((m)->mtx_lock == MTX_UNOWNED)
   82 
   83 #define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED)
   84 
   85 #define mtx_owner(m)    ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
   86 
   87 #ifdef DDB
   88 static void     db_show_mtx(struct lock_object *lock);
   89 #endif
   90 static void     lock_mtx(struct lock_object *lock, int how);
   91 static void     lock_spin(struct lock_object *lock, int how);
   92 static int      unlock_mtx(struct lock_object *lock);
   93 static int      unlock_spin(struct lock_object *lock);
   94 
   95 /*
   96  * Lock classes for sleep and spin mutexes.
   97  */
   98 struct lock_class lock_class_mtx_sleep = {
   99         .lc_name = "sleep mutex",
  100         .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
  101 #ifdef DDB
  102         .lc_ddb_show = db_show_mtx,
  103 #endif
  104         .lc_lock = lock_mtx,
  105         .lc_unlock = unlock_mtx,
  106 };
  107 struct lock_class lock_class_mtx_spin = {
  108         .lc_name = "spin mutex",
  109         .lc_flags = LC_SPINLOCK | LC_RECURSABLE,
  110 #ifdef DDB
  111         .lc_ddb_show = db_show_mtx,
  112 #endif
  113         .lc_lock = lock_spin,
  114         .lc_unlock = unlock_spin,
  115 };
  116 
  117 /*
  118  * System-wide mutexes
  119  */
  120 struct mtx blocked_lock;
  121 struct mtx Giant;
  122 
  123 #ifdef LOCK_PROFILING
  124 static inline void lock_profile_init(void)
  125 {
  126         int i;
  127         /* Initialize the mutex profiling locks */
  128         for (i = 0; i < LPROF_LOCK_SIZE; i++) {
  129                 mtx_init(&lprof_locks[i], "mprof lock",
  130                     NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE);
  131         }
  132 }
  133 #else
  134 static inline void lock_profile_init(void) {;}
  135 #endif
  136 
  137 void
  138 lock_mtx(struct lock_object *lock, int how)
  139 {
  140 
  141         mtx_lock((struct mtx *)lock);
  142 }
  143 
  144 void
  145 lock_spin(struct lock_object *lock, int how)
  146 {
  147 
  148         panic("spin locks can only use msleep_spin");
  149 }
  150 
  151 int
  152 unlock_mtx(struct lock_object *lock)
  153 {
  154         struct mtx *m;
  155 
  156         m = (struct mtx *)lock;
  157         mtx_assert(m, MA_OWNED | MA_NOTRECURSED);
  158         mtx_unlock(m);
  159         return (0);
  160 }
  161 
  162 int
  163 unlock_spin(struct lock_object *lock)
  164 {
  165 
  166         panic("spin locks can only use msleep_spin");
  167 }
  168 
  169 /*
  170  * Function versions of the inlined __mtx_* macros.  These are used by
  171  * modules and can also be called from assembly language if needed.
  172  */
  173 void
  174 _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
  175 {
  176 
  177         MPASS(curthread != NULL);
  178         KASSERT(m->mtx_lock != MTX_DESTROYED,
  179             ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
  180         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  181             ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  182             file, line));
  183         WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
  184             file, line);
  185 
  186         _get_sleep_lock(m, curthread, opts, file, line);
  187         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  188             line);
  189         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  190         curthread->td_locks++;
  191 }
  192 
  193 void
  194 _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
  195 {
  196         MPASS(curthread != NULL);
  197         KASSERT(m->mtx_lock != MTX_DESTROYED,
  198             ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
  199         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  200             ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  201             file, line));
  202         curthread->td_locks--;
  203         WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  204         LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
  205             line);
  206         mtx_assert(m, MA_OWNED);
  207 
  208         if (m->mtx_recurse == 0)
  209                 lock_profile_release_lock(&m->lock_object);
  210         _rel_sleep_lock(m, curthread, opts, file, line);
  211 }
  212 
  213 void
  214 _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
  215 {
  216 
  217         MPASS(curthread != NULL);
  218         KASSERT(m->mtx_lock != MTX_DESTROYED,
  219             ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
  220         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  221             ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
  222             m->lock_object.lo_name, file, line));
  223         if (mtx_owned(m))
  224                 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  225             ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n",
  226                     m->lock_object.lo_name, file, line));
  227         WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
  228             file, line);
  229         _get_spin_lock(m, curthread, opts, file, line);
  230         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  231             line);
  232         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  233 }
  234 
  235 void
  236 _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
  237 {
  238 
  239         MPASS(curthread != NULL);
  240         KASSERT(m->mtx_lock != MTX_DESTROYED,
  241             ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
  242         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  243             ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
  244             m->lock_object.lo_name, file, line));
  245         WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  246         LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,
  247             line);
  248         mtx_assert(m, MA_OWNED);
  249 
  250         _rel_spin_lock(m);
  251 }
  252 
  253 /*
  254  * The important part of mtx_trylock{,_flags}()
  255  * Tries to acquire lock `m.'  If this function is called on a mutex that
  256  * is already owned, it will recursively acquire the lock.
  257  */
  258 int
  259 _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
  260 {
  261         int rval, contested = 0;
  262         uint64_t waittime = 0;
  263         
  264         MPASS(curthread != NULL);
  265         KASSERT(m->mtx_lock != MTX_DESTROYED,
  266             ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
  267         KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
  268             ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
  269             file, line));
  270 
  271         if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {
  272                 m->mtx_recurse++;
  273                 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
  274                 rval = 1;
  275         } else
  276                 rval = _obtain_lock(m, (uintptr_t)curthread);
  277 
  278         LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
  279         if (rval) {
  280                 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
  281                     file, line);
  282                 curthread->td_locks++;
  283                 if (m->mtx_recurse == 0)
  284                         lock_profile_obtain_lock_success(&m->lock_object, contested,
  285                             waittime, file, line);
  286 
  287         }
  288 
  289         return (rval);
  290 }
  291 
  292 /*
  293  * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
  294  *
  295  * We call this if the lock is either contested (i.e. we need to go to
  296  * sleep waiting for it), or if we need to recurse on it.
  297  */
  298 void
  299 _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
  300     int line)
  301 {
  302         struct turnstile *ts;
  303 #ifdef ADAPTIVE_MUTEXES
  304         volatile struct thread *owner;
  305 #endif
  306 #ifdef KTR
  307         int cont_logged = 0;
  308 #endif
  309         int contested = 0;
  310         uint64_t waittime = 0;
  311         uintptr_t v;
  312         
  313         if (mtx_owned(m)) {
  314                 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  315             ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
  316                     m->lock_object.lo_name, file, line));
  317                 m->mtx_recurse++;
  318                 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
  319                 if (LOCK_LOG_TEST(&m->lock_object, opts))
  320                         CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
  321                 return;
  322         }
  323 
  324         lock_profile_obtain_lock_failed(&m->lock_object,
  325                     &contested, &waittime);
  326         if (LOCK_LOG_TEST(&m->lock_object, opts))
  327                 CTR4(KTR_LOCK,
  328                     "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
  329                     m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
  330 
  331         while (!_obtain_lock(m, tid)) { 
  332 #ifdef ADAPTIVE_MUTEXES
  333                 /*
  334                  * If the owner is running on another CPU, spin until the
  335                  * owner stops running or the state of the lock changes.
  336                  */
  337                 v = m->mtx_lock;
  338                 if (v != MTX_UNOWNED) {
  339                         owner = (struct thread *)(v & ~MTX_FLAGMASK);
  340 #ifdef ADAPTIVE_GIANT
  341                         if (TD_IS_RUNNING(owner)) {
  342 #else
  343                         if (m != &Giant && TD_IS_RUNNING(owner)) {
  344 #endif
  345                                 if (LOCK_LOG_TEST(&m->lock_object, 0))
  346                                         CTR3(KTR_LOCK,
  347                                             "%s: spinning on %p held by %p",
  348                                             __func__, m, owner);
  349                                 while (mtx_owner(m) == owner &&
  350                                     TD_IS_RUNNING(owner))
  351                                         cpu_spinwait();
  352                                 continue;
  353                         }
  354                 }
  355 #endif
  356 
  357                 ts = turnstile_trywait(&m->lock_object);
  358                 v = m->mtx_lock;
  359 
  360                 /*
  361                  * Check if the lock has been released while spinning for
  362                  * the turnstile chain lock.
  363                  */
  364                 if (v == MTX_UNOWNED) {
  365                         turnstile_cancel(ts);
  366                         cpu_spinwait();
  367                         continue;
  368                 }
  369 
  370                 MPASS(v != MTX_CONTESTED);
  371 
  372 #ifdef ADAPTIVE_MUTEXES
  373                 /*
  374                  * If the current owner of the lock is executing on another
  375                  * CPU quit the hard path and try to spin.
  376                  */
  377                 owner = (struct thread *)(v & ~MTX_FLAGMASK);
  378 #ifdef ADAPTIVE_GIANT
  379                 if (TD_IS_RUNNING(owner)) {
  380 #else
  381                 if (m != &Giant && TD_IS_RUNNING(owner)) {
  382 #endif
  383                         turnstile_cancel(ts);
  384                         cpu_spinwait();
  385                         continue;
  386                 }
  387 #endif
  388 
  389                 /*
  390                  * If the mutex isn't already contested and a failure occurs
  391                  * setting the contested bit, the mutex was either released
  392                  * or the state of the MTX_RECURSED bit changed.
  393                  */
  394                 if ((v & MTX_CONTESTED) == 0 &&
  395                     !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
  396                         turnstile_cancel(ts);
  397                         cpu_spinwait();
  398                         continue;
  399                 }
  400 
  401                 /*
  402                  * We definitely must sleep for this lock.
  403                  */
  404                 mtx_assert(m, MA_NOTOWNED);
  405 
  406 #ifdef KTR
  407                 if (!cont_logged) {
  408                         CTR6(KTR_CONTENTION,
  409                             "contention: %p at %s:%d wants %s, taken by %s:%d",
  410                             (void *)tid, file, line, m->lock_object.lo_name,
  411                             WITNESS_FILE(&m->lock_object),
  412                             WITNESS_LINE(&m->lock_object));
  413                         cont_logged = 1;
  414                 }
  415 #endif
  416 
  417                 /*
  418                  * Block on the turnstile.
  419                  */
  420                 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
  421         }
  422 #ifdef KTR
  423         if (cont_logged) {
  424                 CTR4(KTR_CONTENTION,
  425                     "contention end: %s acquired by %p at %s:%d",
  426                     m->lock_object.lo_name, (void *)tid, file, line);
  427         }
  428 #endif
  429         lock_profile_obtain_lock_success(&m->lock_object, contested,    
  430             waittime, (file), (line));                                  
  431 }
  432 
  433 static void
  434 _mtx_lock_spin_failed(struct mtx *m)
  435 {
  436         struct thread *td;
  437 
  438         td = mtx_owner(m);
  439 
  440         /* If the mutex is unlocked, try again. */
  441         if (td == NULL)
  442                 return;
  443 
  444         printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
  445             m, m->lock_object.lo_name, td, td->td_tid);
  446 #ifdef WITNESS
  447         witness_display_spinlock(&m->lock_object, td);
  448 #endif
  449         panic("spin lock held too long");
  450 }
  451 
  452 #ifdef SMP
  453 /*
  454  * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
  455  *
  456  * This is only called if we need to actually spin for the lock. Recursion
  457  * is handled inline.
  458  */
  459 void
  460 _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
  461     int line)
  462 {
  463         int i = 0, contested = 0;
  464         uint64_t waittime = 0;
  465         
  466         if (LOCK_LOG_TEST(&m->lock_object, opts))
  467                 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
  468 
  469         lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
  470         while (!_obtain_lock(m, tid)) {
  471 
  472                 /* Give interrupts a chance while we spin. */
  473                 spinlock_exit();
  474                 while (m->mtx_lock != MTX_UNOWNED) {
  475                         if (i++ < 10000000) {
  476                                 cpu_spinwait();
  477                                 continue;
  478                         }
  479                         if (i < 60000000 || kdb_active || panicstr != NULL)
  480                                 DELAY(1);
  481                         else
  482                                 _mtx_lock_spin_failed(m);
  483                         cpu_spinwait();
  484                 }
  485                 spinlock_enter();
  486         }
  487 
  488         if (LOCK_LOG_TEST(&m->lock_object, opts))
  489                 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
  490 
  491         lock_profile_obtain_lock_success(&m->lock_object, contested,    
  492             waittime, (file), (line));
  493 }
  494 #endif /* SMP */
  495 
  496 void
  497 _thread_lock_flags(struct thread *td, int opts, const char *file, int line)
  498 {
  499         struct mtx *m;
  500         uintptr_t tid;
  501         int i, contested;
  502         uint64_t waittime;
  503 
  504         contested = i = 0;
  505         waittime = 0;
  506         tid = (uintptr_t)curthread;
  507         for (;;) {
  508 retry:
  509                 spinlock_enter();
  510                 m = td->td_lock;
  511                 KASSERT(m->mtx_lock != MTX_DESTROYED,
  512                     ("thread_lock() of destroyed mutex @ %s:%d", file, line));
  513                 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
  514                     ("thread_lock() of sleep mutex %s @ %s:%d",
  515                     m->lock_object.lo_name, file, line));
  516                 if (mtx_owned(m))
  517                         KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
  518             ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
  519                             m->lock_object.lo_name, file, line));
  520                 WITNESS_CHECKORDER(&m->lock_object,
  521                     opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line);
  522                 while (!_obtain_lock(m, tid)) {
  523                         if (m->mtx_lock == tid) {
  524                                 m->mtx_recurse++;
  525                                 break;
  526                         }
  527                         lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
  528                         /* Give interrupts a chance while we spin. */
  529                         spinlock_exit();
  530                         while (m->mtx_lock != MTX_UNOWNED) {
  531                                 if (i++ < 10000000)
  532                                         cpu_spinwait();
  533                                 else if (i < 60000000 ||
  534                                     kdb_active || panicstr != NULL)
  535                                         DELAY(1);
  536                                 else
  537                                         _mtx_lock_spin_failed(m);
  538                                 cpu_spinwait();
  539                                 if (m != td->td_lock)
  540                                         goto retry;
  541                         }
  542                         spinlock_enter();
  543                 }
  544                 if (m == td->td_lock)
  545                         break;
  546                 _rel_spin_lock(m);      /* does spinlock_exit() */
  547         }
  548         lock_profile_obtain_lock_success(&m->lock_object, contested,    
  549             waittime, (file), (line));
  550         LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
  551             line);
  552         WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
  553 }
  554 
  555 struct mtx *
  556 thread_lock_block(struct thread *td)
  557 {
  558         struct mtx *lock;
  559 
  560         THREAD_LOCK_ASSERT(td, MA_OWNED);
  561         lock = td->td_lock;
  562         td->td_lock = &blocked_lock;
  563         mtx_unlock_spin(lock);
  564 
  565         return (lock);
  566 }
  567 
  568 void
  569 thread_lock_unblock(struct thread *td, struct mtx *new)
  570 {
  571         mtx_assert(new, MA_OWNED);
  572         MPASS(td->td_lock == &blocked_lock);
  573         atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new);
  574 }
  575 
  576 void
  577 thread_lock_set(struct thread *td, struct mtx *new)
  578 {
  579         struct mtx *lock;
  580 
  581         mtx_assert(new, MA_OWNED);
  582         THREAD_LOCK_ASSERT(td, MA_OWNED);
  583         lock = td->td_lock;
  584         td->td_lock = new;
  585         mtx_unlock_spin(lock);
  586 }
  587 
  588 /*
  589  * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
  590  *
  591  * We are only called here if the lock is recursed or contested (i.e. we
  592  * need to wake up a blocked thread).
  593  */
  594 void
  595 _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
  596 {
  597         struct turnstile *ts;
  598 
  599         if (mtx_recursed(m)) {
  600                 if (--(m->mtx_recurse) == 0)
  601                         atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
  602                 if (LOCK_LOG_TEST(&m->lock_object, opts))
  603                         CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
  604                 return;
  605         }
  606 
  607         /*
  608          * We have to lock the chain before the turnstile so this turnstile
  609          * can be removed from the hash list if it is empty.
  610          */
  611         turnstile_chain_lock(&m->lock_object);
  612         ts = turnstile_lookup(&m->lock_object);
  613         if (LOCK_LOG_TEST(&m->lock_object, opts))
  614                 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
  615 
  616         MPASS(ts != NULL);
  617         turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
  618         _release_lock_quick(m);
  619         /*
  620          * This turnstile is now no longer associated with the mutex.  We can
  621          * unlock the chain lock so a new turnstile may take it's place.
  622          */
  623         turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
  624         turnstile_chain_unlock(&m->lock_object);
  625 }
  626 
  627 /*
  628  * All the unlocking of MTX_SPIN locks is done inline.
  629  * See the _rel_spin_lock() macro for the details.
  630  */
  631 
  632 /*
  633  * The backing function for the INVARIANTS-enabled mtx_assert()
  634  */
  635 #ifdef INVARIANT_SUPPORT
  636 void
  637 _mtx_assert(struct mtx *m, int what, const char *file, int line)
  638 {
  639 
  640         if (panicstr != NULL || dumping)
  641                 return;
  642         switch (what) {
  643         case MA_OWNED:
  644         case MA_OWNED | MA_RECURSED:
  645         case MA_OWNED | MA_NOTRECURSED:
  646                 if (!mtx_owned(m))
  647                         panic("mutex %s not owned at %s:%d",
  648                             m->lock_object.lo_name, file, line);
  649                 if (mtx_recursed(m)) {
  650                         if ((what & MA_NOTRECURSED) != 0)
  651                                 panic("mutex %s recursed at %s:%d",
  652                                     m->lock_object.lo_name, file, line);
  653                 } else if ((what & MA_RECURSED) != 0) {
  654                         panic("mutex %s unrecursed at %s:%d",
  655                             m->lock_object.lo_name, file, line);
  656                 }
  657                 break;
  658         case MA_NOTOWNED:
  659                 if (mtx_owned(m))
  660                         panic("mutex %s owned at %s:%d",
  661                             m->lock_object.lo_name, file, line);
  662                 break;
  663         default:
  664                 panic("unknown mtx_assert at %s:%d", file, line);
  665         }
  666 }
  667 #endif
  668 
  669 /*
  670  * The MUTEX_DEBUG-enabled mtx_validate()
  671  *
  672  * Most of these checks have been moved off into the LO_INITIALIZED flag
  673  * maintained by the witness code.
  674  */
  675 #ifdef MUTEX_DEBUG
  676 
  677 void    mtx_validate(struct mtx *);
  678 
  679 void
  680 mtx_validate(struct mtx *m)
  681 {
  682 
  683 /*
  684  * XXX: When kernacc() does not require Giant we can reenable this check
  685  */
  686 #ifdef notyet
  687         /*
  688          * Can't call kernacc() from early init386(), especially when
  689          * initializing Giant mutex, because some stuff in kernacc()
  690          * requires Giant itself.
  691          */
  692         if (!cold)
  693                 if (!kernacc((caddr_t)m, sizeof(m),
  694                     VM_PROT_READ | VM_PROT_WRITE))
  695                         panic("Can't read and write to mutex %p", m);
  696 #endif
  697 }
  698 #endif
  699 
  700 /*
  701  * General init routine used by the MTX_SYSINIT() macro.
  702  */
  703 void
  704 mtx_sysinit(void *arg)
  705 {
  706         struct mtx_args *margs = arg;
  707 
  708         mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts);
  709 }
  710 
  711 /*
  712  * Mutex initialization routine; initialize lock `m' of type contained in
  713  * `opts' with options contained in `opts' and name `name.'  The optional
  714  * lock type `type' is used as a general lock category name for use with
  715  * witness.
  716  */
  717 void
  718 mtx_init(struct mtx *m, const char *name, const char *type, int opts)
  719 {
  720         struct lock_class *class;
  721         int flags;
  722 
  723         MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
  724                 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0);
  725         ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock,
  726             ("%s: mtx_lock not aligned for %s:%p", __func__, name,
  727             &m->mtx_lock));
  728 
  729 #ifdef MUTEX_DEBUG
  730         /* Diagnostic and error correction */
  731         mtx_validate(m);
  732 #endif
  733 
  734         /* Determine lock class and lock flags. */
  735         if (opts & MTX_SPIN)
  736                 class = &lock_class_mtx_spin;
  737         else
  738                 class = &lock_class_mtx_sleep;
  739         flags = 0;
  740         if (opts & MTX_QUIET)
  741                 flags |= LO_QUIET;
  742         if (opts & MTX_RECURSE)
  743                 flags |= LO_RECURSABLE;
  744         if ((opts & MTX_NOWITNESS) == 0)
  745                 flags |= LO_WITNESS;
  746         if (opts & MTX_DUPOK)
  747                 flags |= LO_DUPOK;
  748         if (opts & MTX_NOPROFILE)
  749                 flags |= LO_NOPROFILE;
  750 
  751         /* Initialize mutex. */
  752         m->mtx_lock = MTX_UNOWNED;
  753         m->mtx_recurse = 0;
  754 
  755         lock_init(&m->lock_object, class, name, type, flags);
  756 }
  757 
  758 /*
  759  * Remove lock `m' from all_mtx queue.  We don't allow MTX_QUIET to be
  760  * passed in as a flag here because if the corresponding mtx_init() was
  761  * called with MTX_QUIET set, then it will already be set in the mutex's
  762  * flags.
  763  */
  764 void
  765 mtx_destroy(struct mtx *m)
  766 {
  767 
  768         if (!mtx_owned(m))
  769                 MPASS(mtx_unowned(m));
  770         else {
  771                 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
  772 
  773                 /* Perform the non-mtx related part of mtx_unlock_spin(). */
  774                 if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin)
  775                         spinlock_exit();
  776                 else
  777                         curthread->td_locks--;
  778 
  779                 /* Tell witness this isn't locked to make it happy. */
  780                 WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
  781                     __LINE__);
  782         }
  783 
  784         m->mtx_lock = MTX_DESTROYED;
  785         lock_destroy(&m->lock_object);
  786 }
  787 
  788 /*
  789  * Intialize the mutex code and system mutexes.  This is called from the MD
  790  * startup code prior to mi_startup().  The per-CPU data space needs to be
  791  * setup before this is called.
  792  */
  793 void
  794 mutex_init(void)
  795 {
  796 
  797         /* Setup turnstiles so that sleep mutexes work. */
  798         init_turnstiles();
  799 
  800         /*
  801          * Initialize mutexes.
  802          */
  803         mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
  804         mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
  805         blocked_lock.mtx_lock = 0xdeadc0de;     /* Always blocked. */
  806         mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
  807         mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE);
  808         mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
  809         mtx_lock(&Giant);
  810         
  811         lock_profile_init();
  812 }
  813 
  814 #ifdef DDB
  815 void
  816 db_show_mtx(struct lock_object *lock)
  817 {
  818         struct thread *td;
  819         struct mtx *m;
  820 
  821         m = (struct mtx *)lock;
  822 
  823         db_printf(" flags: {");
  824         if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
  825                 db_printf("SPIN");
  826         else
  827                 db_printf("DEF");
  828         if (m->lock_object.lo_flags & LO_RECURSABLE)
  829                 db_printf(", RECURSE");
  830         if (m->lock_object.lo_flags & LO_DUPOK)
  831                 db_printf(", DUPOK");
  832         db_printf("}\n");
  833         db_printf(" state: {");
  834         if (mtx_unowned(m))
  835                 db_printf("UNOWNED");
  836         else if (mtx_destroyed(m))
  837                 db_printf("DESTROYED");
  838         else {
  839                 db_printf("OWNED");
  840                 if (m->mtx_lock & MTX_CONTESTED)
  841                         db_printf(", CONTESTED");
  842                 if (m->mtx_lock & MTX_RECURSED)
  843                         db_printf(", RECURSED");
  844         }
  845         db_printf("}\n");
  846         if (!mtx_unowned(m) && !mtx_destroyed(m)) {
  847                 td = mtx_owner(m);
  848                 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td,
  849                     td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
  850                 if (mtx_recursed(m))
  851                         db_printf(" recursed: %d\n", m->mtx_recurse);
  852         }
  853 }
  854 #endif

Cache object: 7a8e7d78e5b245898c9830405339d807


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.