The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_lock.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $     */
    2 
    3 /*-
    4  * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to The NetBSD Foundation
    8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
    9  * NASA Ames Research Center, and by Andrew Doran.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30  * POSSIBILITY OF SUCH DAMAGE.
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/proc.h>
   38 #include <sys/lock.h>
   39 #include <sys/systm.h>
   40 #include <sys/kernel.h>
   41 #include <sys/lockdebug.h>
   42 #include <sys/cpu.h>
   43 #include <sys/syslog.h>
   44 #include <sys/atomic.h>
   45 
   46 #include <machine/stdarg.h>
   47 #include <machine/lock.h>
   48 
   49 #include <dev/lockstat.h>
   50 
   51 #define RETURN_ADDRESS  (uintptr_t)__builtin_return_address(0)
   52 
   53 bool    kernel_lock_dodebug;
   54 
   55 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)]
   56     __aligned(CACHE_LINE_SIZE);
   57 
   58 #if defined(DEBUG) || defined(LKM)
   59 void
   60 assert_sleepable(void)
   61 {
   62         const char *reason;
   63 
   64         if (panicstr != NULL) {
   65                 return;
   66         }
   67 
   68         LOCKDEBUG_BARRIER(kernel_lock, 1);
   69 
   70         reason = NULL;
   71         if (CURCPU_IDLE_P() && !cold) {
   72                 reason = "idle";
   73         }
   74         if (cpu_intr_p()) {
   75                 reason = "interrupt";
   76         }
   77         if ((curlwp->l_pflag & LP_INTR) != 0) {
   78                 reason = "softint";
   79         }
   80 
   81         if (reason) {
   82                 panic("%s: %s caller=%p", __func__, reason,
   83                     (void *)RETURN_ADDRESS);
   84         }
   85 }
   86 #endif /* defined(DEBUG) || defined(LKM) */
   87 
   88 /*
   89  * Functions for manipulating the kernel_lock.  We put them here
   90  * so that they show up in profiles.
   91  */
   92 
   93 #define _KERNEL_LOCK_ABORT(msg)                                         \
   94     LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg)
   95 
   96 #ifdef LOCKDEBUG
   97 #define _KERNEL_LOCK_ASSERT(cond)                                       \
   98 do {                                                                    \
   99         if (!(cond))                                                    \
  100                 _KERNEL_LOCK_ABORT("assertion failed: " #cond);         \
  101 } while (/* CONSTCOND */ 0)
  102 #else
  103 #define _KERNEL_LOCK_ASSERT(cond)       /* nothing */
  104 #endif
  105 
  106 void    _kernel_lock_dump(volatile void *);
  107 
  108 lockops_t _kernel_lock_ops = {
  109         "Kernel lock",
  110         LOCKOPS_SPIN,
  111         _kernel_lock_dump
  112 };
  113 
  114 /*
  115  * Initialize the kernel lock.
  116  */
  117 void
  118 kernel_lock_init(void)
  119 {
  120 
  121         CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t));
  122         __cpu_simple_lock_init(kernel_lock);
  123         kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops,
  124             RETURN_ADDRESS);
  125 }
  126 
  127 /*
  128  * Print debugging information about the kernel lock.
  129  */
  130 void
  131 _kernel_lock_dump(volatile void *junk)
  132 {
  133         struct cpu_info *ci = curcpu();
  134 
  135         (void)junk;
  136 
  137         printf_nolog("curcpu holds : %18d wanted by: %#018lx\n",
  138             ci->ci_biglock_count, (long)ci->ci_biglock_wanted);
  139 }
  140 
  141 /*
  142  * Acquire 'nlocks' holds on the kernel lock.  If 'l' is non-null, the
  143  * acquisition is from process context.
  144  */
  145 void
  146 _kernel_lock(int nlocks)
  147 {
  148         struct cpu_info *ci;
  149         LOCKSTAT_TIMER(spintime);
  150         LOCKSTAT_FLAG(lsflag);
  151         struct lwp *owant;
  152         u_int spins;
  153         int s;
  154         struct lwp *l = curlwp;
  155 
  156         _KERNEL_LOCK_ASSERT(nlocks > 0);
  157 
  158         s = splvm();
  159         ci = curcpu();
  160         if (ci->ci_biglock_count != 0) {
  161                 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
  162                 ci->ci_biglock_count += nlocks;
  163                 l->l_blcnt += nlocks;
  164                 splx(s);
  165                 return;
  166         }
  167 
  168         _KERNEL_LOCK_ASSERT(l->l_blcnt == 0);
  169         LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS,
  170             false, false);
  171 
  172         if (__cpu_simple_lock_try(kernel_lock)) {
  173                 ci->ci_biglock_count = nlocks;
  174                 l->l_blcnt = nlocks;
  175                 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
  176                     RETURN_ADDRESS, 0);
  177                 splx(s);
  178                 return;
  179         }
  180 
  181         /*
  182          * To remove the ordering constraint between adaptive mutexes
  183          * and kernel_lock we must make it appear as if this thread is
  184          * blocking.  For non-interlocked mutex release, a store fence
  185          * is required to ensure that the result of any mutex_exit()
  186          * by the current LWP becomes visible on the bus before the set
  187          * of ci->ci_biglock_wanted becomes visible.
  188          */
  189         membar_producer();
  190         owant = ci->ci_biglock_wanted;
  191         ci->ci_biglock_wanted = l;
  192 
  193         /*
  194          * Spin until we acquire the lock.  Once we have it, record the
  195          * time spent with lockstat.
  196          */
  197         LOCKSTAT_ENTER(lsflag);
  198         LOCKSTAT_START_TIMER(lsflag, spintime);
  199 
  200         spins = 0;
  201         do {
  202                 splx(s);
  203                 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) {
  204                         if (SPINLOCK_SPINOUT(spins)) {
  205                                 extern int start_init_exec;
  206                                 if (!start_init_exec)
  207                                         _KERNEL_LOCK_ABORT("spinout");
  208                         }
  209                         SPINLOCK_BACKOFF_HOOK;
  210                         SPINLOCK_SPIN_HOOK;
  211                 }
  212                 s = splvm();
  213         } while (!__cpu_simple_lock_try(kernel_lock));
  214 
  215         ci->ci_biglock_count = nlocks;
  216         l->l_blcnt = nlocks;
  217         LOCKSTAT_STOP_TIMER(lsflag, spintime);
  218         LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
  219             RETURN_ADDRESS, 0);
  220         if (owant == NULL) {
  221                 LOCKSTAT_EVENT_RA(lsflag, kernel_lock,
  222                     LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS);
  223         }
  224         LOCKSTAT_EXIT(lsflag);
  225         splx(s);
  226 
  227         /*
  228          * Now that we have kernel_lock, reset ci_biglock_wanted.  This
  229          * store must be unbuffered (immediately visible on the bus) in
  230          * order for non-interlocked mutex release to work correctly. 
  231          * It must be visible before a mutex_exit() can execute on this
  232          * processor.
  233          *
  234          * Note: only where CAS is available in hardware will this be
  235          * an unbuffered write, but non-interlocked release cannot be
  236          * done on CPUs without CAS in hardware.
  237          */
  238         (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant);
  239 
  240         /*
  241          * Issue a memory barrier as we have acquired a lock.  This also
  242          * prevents stores from a following mutex_exit() being reordered
  243          * to occur before our store to ci_biglock_wanted above.
  244          */
  245         membar_enter();
  246 }
  247 
  248 /*
  249  * Release 'nlocks' holds on the kernel lock.  If 'nlocks' is zero, release
  250  * all holds.  If 'l' is non-null, the release is from process context.
  251  */
  252 void
  253 _kernel_unlock(int nlocks, int *countp)
  254 {
  255         struct cpu_info *ci;
  256         u_int olocks;
  257         int s;
  258         struct lwp *l = curlwp;
  259 
  260         _KERNEL_LOCK_ASSERT(nlocks < 2);
  261 
  262         olocks = l->l_blcnt;
  263 
  264         if (olocks == 0) {
  265                 _KERNEL_LOCK_ASSERT(nlocks <= 0);
  266                 if (countp != NULL)
  267                         *countp = 0;
  268                 return;
  269         }
  270 
  271         _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
  272 
  273         if (nlocks == 0)
  274                 nlocks = olocks;
  275         else if (nlocks == -1) {
  276                 nlocks = 1;
  277                 _KERNEL_LOCK_ASSERT(olocks == 1);
  278         }
  279         s = splvm();
  280         ci = curcpu();
  281         _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt);
  282         if (ci->ci_biglock_count == nlocks) {
  283                 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock,
  284                     RETURN_ADDRESS, 0);
  285                 ci->ci_biglock_count = 0;
  286                 __cpu_simple_unlock(kernel_lock);
  287                 l->l_blcnt -= nlocks;
  288                 splx(s);
  289                 if (l->l_dopreempt)
  290                         kpreempt(0);
  291         } else {
  292                 ci->ci_biglock_count -= nlocks;
  293                 l->l_blcnt -= nlocks;
  294                 splx(s);
  295         }
  296 
  297         if (countp != NULL)
  298                 *countp = olocks;
  299 }

Cache object: 4937ebf794a3290c774998c38e601c23


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.