The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/lwkt_serialize.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
    3  * 
    4  * This code is derived from software contributed to The DragonFly Project
    5  * by Matthew Dillon <dillon@backplane.com>
    6  * 
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in
   15  *    the documentation and/or other materials provided with the
   16  *    distribution.
   17  * 3. Neither the name of The DragonFly Project nor the names of its
   18  *    contributors may be used to endorse or promote products derived
   19  *    from this software without specific, prior written permission.
   20  * 
   21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
   25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
   26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
   27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   32  * SUCH DAMAGE.
   33  */
   34 /*
   35  * This API provides a fast locked-bus-cycle-based serializer.  It's
   36  * basically a low level NON-RECURSIVE exclusive lock that can be held across
   37  * a blocking condition.  It is NOT a mutex.
   38  *
   39  * This serializer is primarily designed for low level situations and
   40  * interrupt/device interaction.  There are two primary facilities.  First,
   41  * the serializer facility itself.  Second, an integrated interrupt handler 
   42  * disablement facility.
   43  */
   44 
   45 #include <sys/param.h>
   46 #include <sys/systm.h>
   47 #include <sys/kernel.h>
   48 #include <sys/proc.h>
   49 #include <sys/rtprio.h>
   50 #include <sys/queue.h>
   51 #include <sys/thread2.h>
   52 #include <sys/serialize.h>
   53 #include <sys/sysctl.h>
   54 #include <sys/ktr.h>
   55 #include <sys/kthread.h>
   56 #include <machine/cpu.h>
   57 #include <machine/cpufunc.h>
   58 #include <machine/specialreg.h>
   59 #include <sys/lock.h>
   60 
   61 struct exp_backoff {
   62         int backoff;
   63         int round;
   64         lwkt_serialize_t s;
   65 };
   66 
   67 #define SLZ_KTR_STRING          "slz=%p"
   68 #define SLZ_KTR_ARGS            lwkt_serialize_t slz
   69 
   70 #ifndef KTR_SERIALIZER
   71 #define KTR_SERIALIZER  KTR_ALL
   72 #endif
   73 
   74 KTR_INFO_MASTER(slz);
   75 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   76 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   77 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   78 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   79 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   80 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   81 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   82 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   83 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   84 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
   85          "slz=%p bo1=%d bo=%d", lwkt_serialize_t slz, int backoff1, int backoff);
   86 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   87 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);
   88 
   89 #define logslz(name, slz)               KTR_LOG(slz_ ## name, slz)
   90 #define logslz_spinbo(slz, bo1, bo)     KTR_LOG(slz_spinbo, slz, bo1, bo)
   91 
   92 static void lwkt_serialize_sleep(void *info);
   93 static void lwkt_serialize_wakeup(void *info);
   94 static void lwkt_serialize_adaptive_sleep(void *bo);
   95 
   96 static int slz_backoff_limit = 128;
   97 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
   98     &slz_backoff_limit, 0, "Backoff limit");
   99 
  100 static int slz_backoff_shift = 1;
  101 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
  102     &slz_backoff_shift, 0, "Backoff shift");
  103 
  104 static int slz_backoff_round;
  105 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
  106 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
  107     &slz_backoff_round, 0,
  108     "Backoff rounding");
  109 
  110 void
  111 lwkt_serialize_init(lwkt_serialize_t s)
  112 {
  113     atomic_intr_init(&s->interlock);
  114 #ifdef INVARIANTS
  115     s->last_td = (void *)-4;
  116 #endif
  117 }
  118 
  119 void
  120 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
  121 {
  122     struct exp_backoff bo;
  123 
  124     bo.backoff = 1;
  125     bo.round = 0;
  126     bo.s = s;
  127 
  128     ASSERT_NOT_SERIALIZED(s);
  129 
  130     logslz(enter_beg, s);
  131     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
  132     logslz(enter_end, s);
  133 #ifdef INVARIANTS
  134     s->last_td = curthread;
  135 #endif
  136 }
  137 
  138 void
  139 lwkt_serialize_enter(lwkt_serialize_t s)
  140 {
  141     ASSERT_NOT_SERIALIZED(s);
  142 
  143     logslz(enter_beg, s);
  144     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
  145     logslz(enter_end, s);
  146 #ifdef INVARIANTS
  147     s->last_td = curthread;
  148 #endif
  149 }
  150 
  151 /*
  152  * Returns non-zero on success
  153  */
  154 int
  155 lwkt_serialize_try(lwkt_serialize_t s)
  156 {
  157     int error;
  158 
  159     ASSERT_NOT_SERIALIZED(s);
  160 
  161     logslz(try, s);
  162     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
  163 #ifdef INVARIANTS
  164         s->last_td = curthread;
  165 #endif
  166         logslz(tryok, s);
  167         return(1);
  168     }
  169     logslz(tryfail, s);
  170     return (0);
  171 }
  172 
  173 void
  174 lwkt_serialize_exit(lwkt_serialize_t s)
  175 {
  176     ASSERT_SERIALIZED(s);
  177 #ifdef INVARIANTS
  178     s->last_td = (void *)-2;
  179 #endif
  180     logslz(exit_beg, s);
  181     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
  182     logslz(exit_end, s);
  183 }
  184 
  185 /*
  186  * Interrupt handler disablement support, used by drivers.  Non-stackable
  187  * (uses bit 30).
  188  */
  189 void
  190 lwkt_serialize_handler_disable(lwkt_serialize_t s)
  191 {
  192     atomic_intr_handler_disable(&s->interlock);
  193 }
  194 
  195 void
  196 lwkt_serialize_handler_enable(lwkt_serialize_t s)
  197 {
  198     atomic_intr_handler_enable(&s->interlock);
  199 }
  200 
  201 void
  202 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *), 
  203                             void *arg, void *frame)
  204 {
  205     /*
  206      * note: a return value of 0 indicates that the interrupt handler is 
  207      * enabled.
  208      */
  209     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
  210         logslz(enter_beg, s);
  211         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
  212         logslz(enter_end, s);
  213 #ifdef INVARIANTS
  214         s->last_td = curthread;
  215 #endif
  216         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
  217             func(arg, frame);
  218 
  219         ASSERT_SERIALIZED(s);
  220 #ifdef INVARIANTS
  221         s->last_td = (void *)-2;
  222 #endif
  223         logslz(exit_beg, s);
  224         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
  225         logslz(exit_end, s);
  226     }
  227 }
  228 
  229 /*
  230  * Similar to handler_call but does not block.  Returns 0 on success, 
  231  * and 1 on failure.
  232  */
  233 int
  234 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
  235                            void *arg, void *frame)
  236 {
  237     /*
  238      * note: a return value of 0 indicates that the interrupt handler is 
  239      * enabled.
  240      */
  241     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
  242         logslz(try, s);
  243         if (atomic_intr_cond_try(&s->interlock) == 0) {
  244 #ifdef INVARIANTS
  245             s->last_td = curthread;
  246 #endif
  247             logslz(tryok, s);
  248 
  249             func(arg, frame);
  250 
  251             ASSERT_SERIALIZED(s);
  252 #ifdef INVARIANTS
  253             s->last_td = (void *)-2;
  254 #endif
  255             logslz(exit_beg, s);
  256             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
  257             logslz(exit_end, s);
  258             return(0);
  259         }
  260     }
  261     logslz(tryfail, s);
  262     return(1);
  263 }
  264 
  265 
  266 /*
  267  * Helper functions
  268  *
  269  * It is possible to race an interrupt which acquires and releases the
  270  * bit, then calls wakeup before we actually go to sleep, so we
  271  * need to check that the interlock is still acquired from within
  272  * a critical section prior to sleeping.
  273  */
  274 static void
  275 lwkt_serialize_sleep(void *info)
  276 {
  277     lwkt_serialize_t s = info;
  278 
  279     tsleep_interlock(s, 0);
  280     if (atomic_intr_cond_test(&s->interlock) != 0) {
  281         logslz(sleep_beg, s);
  282         tsleep(s, PINTERLOCKED, "slize", 0);
  283         logslz(sleep_end, s);
  284     }
  285 }
  286 
  287 static void
  288 lwkt_serialize_adaptive_sleep(void *arg)
  289 {
  290     struct exp_backoff *bo = arg;
  291     lwkt_serialize_t s = bo->s;
  292     int backoff;
  293 
  294     /*
  295      * Randomize backoff value
  296      */
  297 #ifdef _RDTSC_SUPPORTED_
  298     if (cpu_feature & CPUID_TSC) {
  299         backoff =
  300         (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
  301          (bo->backoff - 1)) + 1;
  302     } else
  303 #endif
  304         backoff = bo->backoff;
  305 
  306     logslz_spinbo(s, bo->backoff, backoff);
  307 
  308     /*
  309      * Quick backoff
  310      */
  311     for (; backoff; --backoff)
  312         cpu_pause();
  313     if (bo->backoff < slz_backoff_limit) {
  314         bo->backoff <<= slz_backoff_shift;
  315         return;
  316     } else {
  317         bo->backoff = 1;
  318         bo->round++;
  319         if (bo->round >= slz_backoff_round)
  320             bo->round = 0;
  321         else
  322             return;
  323     }
  324 
  325     tsleep_interlock(s, 0);
  326     if (atomic_intr_cond_test(&s->interlock) != 0) {
  327         logslz(sleep_beg, s);
  328         tsleep(s, PINTERLOCKED, "slize", 0);
  329         logslz(sleep_end, s);
  330     }
  331 }
  332 
  333 static void
  334 lwkt_serialize_wakeup(void *info)
  335 {
  336     logslz(wakeup_beg, info);
  337     wakeup(info);
  338     logslz(wakeup_end, info);
  339 }
  340 
  341 static void
  342 lwkt_serialize_sysinit(void *dummy __unused)
  343 {
  344         if (slz_backoff_round <= 0)
  345                 slz_backoff_round = ncpus * 2;
  346 }
  347 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
  348         lwkt_serialize_sysinit, NULL);

Cache object: b0223b2fc1a95e143935ead02c88b4d4


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.