The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_synch.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-3-Clause
    3  *
    4  * Copyright (c) 1982, 1986, 1990, 1991, 1993
    5  *      The Regents of the University of California.  All rights reserved.
    6  * (c) UNIX System Laboratories, Inc.
    7  * All or some portions of this file are derived from material licensed
    8  * to the University of California by American Telephone and Telegraph
    9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
   10  * the permission of UNIX System Laboratories, Inc.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. Neither the name of the University nor the names of its contributors
   21  *    may be used to endorse or promote products derived from this software
   22  *    without specific prior written permission.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   34  * SUCH DAMAGE.
   35  *
   36  *      @(#)kern_synch.c        8.9 (Berkeley) 5/19/95
   37  */
   38 
   39 #include <sys/cdefs.h>
   40 __FBSDID("$FreeBSD$");
   41 
   42 #include "opt_ktrace.h"
   43 #include "opt_sched.h"
   44 
   45 #include <sys/param.h>
   46 #include <sys/systm.h>
   47 #include <sys/blockcount.h>
   48 #include <sys/condvar.h>
   49 #include <sys/kdb.h>
   50 #include <sys/kernel.h>
   51 #include <sys/ktr.h>
   52 #include <sys/lock.h>
   53 #include <sys/mutex.h>
   54 #include <sys/proc.h>
   55 #include <sys/resourcevar.h>
   56 #include <sys/sched.h>
   57 #include <sys/sdt.h>
   58 #include <sys/signalvar.h>
   59 #include <sys/sleepqueue.h>
   60 #include <sys/smp.h>
   61 #include <sys/sx.h>
   62 #include <sys/sysctl.h>
   63 #include <sys/sysproto.h>
   64 #include <sys/vmmeter.h>
   65 #ifdef KTRACE
   66 #include <sys/uio.h>
   67 #include <sys/ktrace.h>
   68 #endif
   69 #ifdef EPOCH_TRACE
   70 #include <sys/epoch.h>
   71 #endif
   72 
   73 #include <machine/cpu.h>
   74 
   75 static void synch_setup(void *dummy);
   76 SYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup,
   77     NULL);
   78 
   79 int     hogticks;
   80 static const char pause_wchan[MAXCPU];
   81 
   82 static struct callout loadav_callout;
   83 
   84 struct loadavg averunnable =
   85         { {0, 0, 0}, FSCALE };  /* load average, of runnable procs */
   86 /*
   87  * Constants for averages over 1, 5, and 15 minutes
   88  * when sampling at 5 second intervals.
   89  */
   90 static fixpt_t cexp[3] = {
   91         0.9200444146293232 * FSCALE,    /* exp(-1/12) */
   92         0.9834714538216174 * FSCALE,    /* exp(-1/60) */
   93         0.9944598480048967 * FSCALE,    /* exp(-1/180) */
   94 };
   95 
   96 /* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */
   97 SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE,
   98     "Fixed-point scale factor used for calculating load average values");
   99 
  100 static void     loadav(void *arg);
  101 
  102 SDT_PROVIDER_DECLARE(sched);
  103 SDT_PROBE_DEFINE(sched, , , preempt);
  104 
  105 static void
  106 sleepinit(void *unused)
  107 {
  108 
  109         hogticks = (hz / 10) * 2;       /* Default only. */
  110         init_sleepqueues();
  111 }
  112 
  113 /*
  114  * vmem tries to lock the sleepq mutexes when free'ing kva, so make sure
  115  * it is available.
  116  */
  117 SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, NULL);
  118 
  119 /*
  120  * General sleep call.  Suspends the current thread until a wakeup is
  121  * performed on the specified identifier.  The thread will then be made
  122  * runnable with the specified priority.  Sleeps at most sbt units of time
  123  * (0 means no timeout).  If pri includes the PCATCH flag, let signals
  124  * interrupt the sleep, otherwise ignore them while sleeping.  Returns 0 if
  125  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
  126  * signal becomes pending, ERESTART is returned if the current system
  127  * call should be restarted if possible, and EINTR is returned if the system
  128  * call should be interrupted by the signal (return EINTR).
  129  *
  130  * The lock argument is unlocked before the caller is suspended, and
  131  * re-locked before _sleep() returns.  If priority includes the PDROP
  132  * flag the lock is not re-locked before returning.
  133  */
  134 int
  135 _sleep(const void *ident, struct lock_object *lock, int priority,
  136     const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
  137 {
  138         struct thread *td;
  139         struct lock_class *class;
  140         uintptr_t lock_state;
  141         int catch, pri, rval, sleepq_flags;
  142         WITNESS_SAVE_DECL(lock_witness);
  143 
  144         td = curthread;
  145 #ifdef KTRACE
  146         if (KTRPOINT(td, KTR_CSW))
  147                 ktrcsw(1, 0, wmesg);
  148 #endif
  149         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
  150             "Sleeping on \"%s\"", wmesg);
  151         KASSERT(sbt != 0 || mtx_owned(&Giant) || lock != NULL,
  152             ("sleeping without a lock"));
  153         KASSERT(ident != NULL, ("_sleep: NULL ident"));
  154         KASSERT(TD_IS_RUNNING(td), ("_sleep: curthread not running"));
  155         if (priority & PDROP)
  156                 KASSERT(lock != NULL && lock != &Giant.lock_object,
  157                     ("PDROP requires a non-Giant lock"));
  158         if (lock != NULL)
  159                 class = LOCK_CLASS(lock);
  160         else
  161                 class = NULL;
  162 
  163         if (SCHEDULER_STOPPED_TD(td)) {
  164                 if (lock != NULL && priority & PDROP)
  165                         class->lc_unlock(lock);
  166                 return (0);
  167         }
  168         catch = priority & PCATCH;
  169         pri = priority & PRIMASK;
  170 
  171         KASSERT(!TD_ON_SLEEPQ(td), ("recursive sleep"));
  172 
  173         if ((uintptr_t)ident >= (uintptr_t)&pause_wchan[0] &&
  174             (uintptr_t)ident <= (uintptr_t)&pause_wchan[MAXCPU - 1])
  175                 sleepq_flags = SLEEPQ_PAUSE;
  176         else
  177                 sleepq_flags = SLEEPQ_SLEEP;
  178         if (catch)
  179                 sleepq_flags |= SLEEPQ_INTERRUPTIBLE;
  180 
  181         sleepq_lock(ident);
  182         CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)",
  183             td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
  184 
  185         if (lock == &Giant.lock_object)
  186                 mtx_assert(&Giant, MA_OWNED);
  187         DROP_GIANT();
  188         if (lock != NULL && lock != &Giant.lock_object &&
  189             !(class->lc_flags & LC_SLEEPABLE)) {
  190                 KASSERT(!(class->lc_flags & LC_SPINLOCK),
  191                     ("spin locks can only use msleep_spin"));
  192                 WITNESS_SAVE(lock, lock_witness);
  193                 lock_state = class->lc_unlock(lock);
  194         } else
  195                 /* GCC needs to follow the Yellow Brick Road */
  196                 lock_state = -1;
  197 
  198         /*
  199          * We put ourselves on the sleep queue and start our timeout
  200          * before calling thread_suspend_check, as we could stop there,
  201          * and a wakeup or a SIGCONT (or both) could occur while we were
  202          * stopped without resuming us.  Thus, we must be ready for sleep
  203          * when cursig() is called.  If the wakeup happens while we're
  204          * stopped, then td will no longer be on a sleep queue upon
  205          * return from cursig().
  206          */
  207         sleepq_add(ident, lock, wmesg, sleepq_flags, 0);
  208         if (sbt != 0)
  209                 sleepq_set_timeout_sbt(ident, sbt, pr, flags);
  210         if (lock != NULL && class->lc_flags & LC_SLEEPABLE) {
  211                 sleepq_release(ident);
  212                 WITNESS_SAVE(lock, lock_witness);
  213                 lock_state = class->lc_unlock(lock);
  214                 sleepq_lock(ident);
  215         }
  216         if (sbt != 0 && catch)
  217                 rval = sleepq_timedwait_sig(ident, pri);
  218         else if (sbt != 0)
  219                 rval = sleepq_timedwait(ident, pri);
  220         else if (catch)
  221                 rval = sleepq_wait_sig(ident, pri);
  222         else {
  223                 sleepq_wait(ident, pri);
  224                 rval = 0;
  225         }
  226 #ifdef KTRACE
  227         if (KTRPOINT(td, KTR_CSW))
  228                 ktrcsw(0, 0, wmesg);
  229 #endif
  230         PICKUP_GIANT();
  231         if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) {
  232                 class->lc_lock(lock, lock_state);
  233                 WITNESS_RESTORE(lock, lock_witness);
  234         }
  235         return (rval);
  236 }
  237 
  238 int
  239 msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg,
  240     sbintime_t sbt, sbintime_t pr, int flags)
  241 {
  242         struct thread *td;
  243         int rval;
  244         WITNESS_SAVE_DECL(mtx);
  245 
  246         td = curthread;
  247         KASSERT(mtx != NULL, ("sleeping without a mutex"));
  248         KASSERT(ident != NULL, ("msleep_spin_sbt: NULL ident"));
  249         KASSERT(TD_IS_RUNNING(td), ("msleep_spin_sbt: curthread not running"));
  250 
  251         if (SCHEDULER_STOPPED_TD(td))
  252                 return (0);
  253 
  254         sleepq_lock(ident);
  255         CTR5(KTR_PROC, "msleep_spin: thread %ld (pid %ld, %s) on %s (%p)",
  256             td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
  257 
  258         DROP_GIANT();
  259         mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
  260         WITNESS_SAVE(&mtx->lock_object, mtx);
  261         mtx_unlock_spin(mtx);
  262 
  263         /*
  264          * We put ourselves on the sleep queue and start our timeout.
  265          */
  266         sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0);
  267         if (sbt != 0)
  268                 sleepq_set_timeout_sbt(ident, sbt, pr, flags);
  269 
  270         /*
  271          * Can't call ktrace with any spin locks held so it can lock the
  272          * ktrace_mtx lock, and WITNESS_WARN considers it an error to hold
  273          * any spin lock.  Thus, we have to drop the sleepq spin lock while
  274          * we handle those requests.  This is safe since we have placed our
  275          * thread on the sleep queue already.
  276          */
  277 #ifdef KTRACE
  278         if (KTRPOINT(td, KTR_CSW)) {
  279                 sleepq_release(ident);
  280                 ktrcsw(1, 0, wmesg);
  281                 sleepq_lock(ident);
  282         }
  283 #endif
  284 #ifdef WITNESS
  285         sleepq_release(ident);
  286         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"",
  287             wmesg);
  288         sleepq_lock(ident);
  289 #endif
  290         if (sbt != 0)
  291                 rval = sleepq_timedwait(ident, 0);
  292         else {
  293                 sleepq_wait(ident, 0);
  294                 rval = 0;
  295         }
  296 #ifdef KTRACE
  297         if (KTRPOINT(td, KTR_CSW))
  298                 ktrcsw(0, 0, wmesg);
  299 #endif
  300         PICKUP_GIANT();
  301         mtx_lock_spin(mtx);
  302         WITNESS_RESTORE(&mtx->lock_object, mtx);
  303         return (rval);
  304 }
  305 
  306 /*
  307  * pause_sbt() delays the calling thread by the given signed binary
  308  * time. During cold bootup, pause_sbt() uses the DELAY() function
  309  * instead of the _sleep() function to do the waiting. The "sbt"
  310  * argument must be greater than or equal to zero. A "sbt" value of
  311  * zero is equivalent to a "sbt" value of one tick.
  312  */
  313 int
  314 pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
  315 {
  316         KASSERT(sbt >= 0, ("pause_sbt: timeout must be >= 0"));
  317 
  318         /* silently convert invalid timeouts */
  319         if (sbt == 0)
  320                 sbt = tick_sbt;
  321 
  322         if ((cold && curthread == &thread0) || kdb_active ||
  323             SCHEDULER_STOPPED()) {
  324                 /*
  325                  * We delay one second at a time to avoid overflowing the
  326                  * system specific DELAY() function(s):
  327                  */
  328                 while (sbt >= SBT_1S) {
  329                         DELAY(1000000);
  330                         sbt -= SBT_1S;
  331                 }
  332                 /* Do the delay remainder, if any */
  333                 sbt = howmany(sbt, SBT_1US);
  334                 if (sbt > 0)
  335                         DELAY(sbt);
  336                 return (EWOULDBLOCK);
  337         }
  338         return (_sleep(&pause_wchan[curcpu], NULL,
  339             (flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags));
  340 }
  341 
  342 /*
  343  * Make all threads sleeping on the specified identifier runnable.
  344  */
  345 void
  346 wakeup(const void *ident)
  347 {
  348         int wakeup_swapper;
  349 
  350         sleepq_lock(ident);
  351         wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0);
  352         sleepq_release(ident);
  353         if (wakeup_swapper) {
  354                 KASSERT(ident != &proc0,
  355                     ("wakeup and wakeup_swapper and proc0"));
  356                 kick_proc0();
  357         }
  358 }
  359 
  360 /*
  361  * Make a thread sleeping on the specified identifier runnable.
  362  * May wake more than one thread if a target thread is currently
  363  * swapped out.
  364  */
  365 void
  366 wakeup_one(const void *ident)
  367 {
  368         int wakeup_swapper;
  369 
  370         sleepq_lock(ident);
  371         wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_DROP, 0, 0);
  372         if (wakeup_swapper)
  373                 kick_proc0();
  374 }
  375 
  376 void
  377 wakeup_any(const void *ident)
  378 {
  379         int wakeup_swapper;
  380 
  381         sleepq_lock(ident);
  382         wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR |
  383             SLEEPQ_DROP, 0, 0);
  384         if (wakeup_swapper)
  385                 kick_proc0();
  386 }
  387 
  388 /*
  389  * Signal sleeping waiters after the counter has reached zero.
  390  */
  391 void
  392 _blockcount_wakeup(blockcount_t *bc, u_int old)
  393 {
  394 
  395         KASSERT(_BLOCKCOUNT_WAITERS(old),
  396             ("%s: no waiters on %p", __func__, bc));
  397 
  398         if (atomic_cmpset_int(&bc->__count, _BLOCKCOUNT_WAITERS_FLAG, 0))
  399                 wakeup(bc);
  400 }
  401 
  402 /*
  403  * Wait for a wakeup or a signal.  This does not guarantee that the count is
  404  * still zero on return.  Callers wanting a precise answer should use
  405  * blockcount_wait() with an interlock.
  406  *
  407  * If there is no work to wait for, return 0.  If the sleep was interrupted by a
  408  * signal, return EINTR or ERESTART, and return EAGAIN otherwise.
  409  */
  410 int
  411 _blockcount_sleep(blockcount_t *bc, struct lock_object *lock, const char *wmesg,
  412     int prio)
  413 {
  414         void *wchan;
  415         uintptr_t lock_state;
  416         u_int old;
  417         int ret;
  418         bool catch, drop;
  419 
  420         KASSERT(lock != &Giant.lock_object,
  421             ("%s: cannot use Giant as the interlock", __func__));
  422 
  423         catch = (prio & PCATCH) != 0;
  424         drop = (prio & PDROP) != 0;
  425         prio &= PRIMASK;
  426 
  427         /*
  428          * Synchronize with the fence in blockcount_release().  If we end up
  429          * waiting, the sleepqueue lock acquisition will provide the required
  430          * side effects.
  431          *
  432          * If there is no work to wait for, but waiters are present, try to put
  433          * ourselves to sleep to avoid jumping ahead.
  434          */
  435         if (atomic_load_acq_int(&bc->__count) == 0) {
  436                 if (lock != NULL && drop)
  437                         LOCK_CLASS(lock)->lc_unlock(lock);
  438                 return (0);
  439         }
  440         lock_state = 0;
  441         wchan = bc;
  442         sleepq_lock(wchan);
  443         DROP_GIANT();
  444         if (lock != NULL)
  445                 lock_state = LOCK_CLASS(lock)->lc_unlock(lock);
  446         old = blockcount_read(bc);
  447         ret = 0;
  448         do {
  449                 if (_BLOCKCOUNT_COUNT(old) == 0) {
  450                         sleepq_release(wchan);
  451                         goto out;
  452                 }
  453                 if (_BLOCKCOUNT_WAITERS(old))
  454                         break;
  455         } while (!atomic_fcmpset_int(&bc->__count, &old,
  456             old | _BLOCKCOUNT_WAITERS_FLAG));
  457         sleepq_add(wchan, NULL, wmesg, catch ? SLEEPQ_INTERRUPTIBLE : 0, 0);
  458         if (catch)
  459                 ret = sleepq_wait_sig(wchan, prio);
  460         else
  461                 sleepq_wait(wchan, prio);
  462         if (ret == 0)
  463                 ret = EAGAIN;
  464 
  465 out:
  466         PICKUP_GIANT();
  467         if (lock != NULL && !drop)
  468                 LOCK_CLASS(lock)->lc_lock(lock, lock_state);
  469 
  470         return (ret);
  471 }
  472 
  473 static void
  474 kdb_switch(void)
  475 {
  476         thread_unlock(curthread);
  477         kdb_backtrace();
  478         kdb_reenter();
  479         panic("%s: did not reenter debugger", __func__);
  480 }
  481 
  482 /*
  483  * The machine independent parts of context switching.
  484  *
  485  * The thread lock is required on entry and is no longer held on return.
  486  */
  487 void
  488 mi_switch(int flags)
  489 {
  490         uint64_t runtime, new_switchtime;
  491         struct thread *td;
  492 
  493         td = curthread;                 /* XXX */
  494         THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
  495         KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code"));
  496 #ifdef INVARIANTS
  497         if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td))
  498                 mtx_assert(&Giant, MA_NOTOWNED);
  499 #endif
  500         KASSERT(td->td_critnest == 1 || KERNEL_PANICKED(),
  501                 ("mi_switch: switch in a critical section"));
  502         KASSERT((flags & (SW_INVOL | SW_VOL)) != 0,
  503             ("mi_switch: switch must be voluntary or involuntary"));
  504 
  505         /*
  506          * Don't perform context switches from the debugger.
  507          */
  508         if (kdb_active)
  509                 kdb_switch();
  510         if (SCHEDULER_STOPPED_TD(td))
  511                 return;
  512         if (flags & SW_VOL) {
  513                 td->td_ru.ru_nvcsw++;
  514                 td->td_swvoltick = ticks;
  515         } else {
  516                 td->td_ru.ru_nivcsw++;
  517                 td->td_swinvoltick = ticks;
  518         }
  519 #ifdef SCHED_STATS
  520         SCHED_STAT_INC(sched_switch_stats[flags & SW_TYPE_MASK]);
  521 #endif
  522         /*
  523          * Compute the amount of time during which the current
  524          * thread was running, and add that to its total so far.
  525          */
  526         new_switchtime = cpu_ticks();
  527         runtime = new_switchtime - PCPU_GET(switchtime);
  528         td->td_runtime += runtime;
  529         td->td_incruntime += runtime;
  530         PCPU_SET(switchtime, new_switchtime);
  531         td->td_generation++;    /* bump preempt-detect counter */
  532         VM_CNT_INC(v_swtch);
  533         PCPU_SET(switchticks, ticks);
  534         CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)",
  535             td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
  536 #ifdef KDTRACE_HOOKS
  537         if (SDT_PROBES_ENABLED() &&
  538             ((flags & SW_PREEMPT) != 0 || ((flags & SW_INVOL) != 0 &&
  539             (flags & SW_TYPE_MASK) == SWT_NEEDRESCHED)))
  540                 SDT_PROBE0(sched, , , preempt);
  541 #endif
  542         sched_switch(td, flags);
  543         CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)",
  544             td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
  545 
  546         /* 
  547          * If the last thread was exiting, finish cleaning it up.
  548          */
  549         if ((td = PCPU_GET(deadthread))) {
  550                 PCPU_SET(deadthread, NULL);
  551                 thread_stash(td);
  552         }
  553         spinlock_exit();
  554 }
  555 
  556 /*
  557  * Change thread state to be runnable, placing it on the run queue if
  558  * it is in memory.  If it is swapped out, return true so our caller
  559  * will know to awaken the swapper.
  560  *
  561  * Requires the thread lock on entry, drops on exit.
  562  */
  563 int
  564 setrunnable(struct thread *td, int srqflags)
  565 {
  566         int swapin;
  567 
  568         THREAD_LOCK_ASSERT(td, MA_OWNED);
  569         KASSERT(td->td_proc->p_state != PRS_ZOMBIE,
  570             ("setrunnable: pid %d is a zombie", td->td_proc->p_pid));
  571 
  572         swapin = 0;
  573         switch (td->td_state) {
  574         case TDS_RUNNING:
  575         case TDS_RUNQ:
  576                 break;
  577         case TDS_CAN_RUN:
  578                 KASSERT((td->td_flags & TDF_INMEM) != 0,
  579                     ("setrunnable: td %p not in mem, flags 0x%X inhibit 0x%X",
  580                     td, td->td_flags, td->td_inhibitors));
  581                 /* unlocks thread lock according to flags */
  582                 sched_wakeup(td, srqflags);
  583                 return (0);
  584         case TDS_INHIBITED:
  585                 /*
  586                  * If we are only inhibited because we are swapped out
  587                  * arrange to swap in this process.
  588                  */
  589                 if (td->td_inhibitors == TDI_SWAPPED &&
  590                     (td->td_flags & TDF_SWAPINREQ) == 0) {
  591                         td->td_flags |= TDF_SWAPINREQ;
  592                         swapin = 1;
  593                 }
  594                 break;
  595         default:
  596                 panic("setrunnable: state 0x%x", td->td_state);
  597         }
  598         if ((srqflags & (SRQ_HOLD | SRQ_HOLDTD)) == 0)
  599                 thread_unlock(td);
  600 
  601         return (swapin);
  602 }
  603 
  604 /*
  605  * Compute a tenex style load average of a quantity on
  606  * 1, 5 and 15 minute intervals.
  607  */
  608 static void
  609 loadav(void *arg)
  610 {
  611         int i, nrun;
  612         struct loadavg *avg;
  613 
  614         nrun = sched_load();
  615         avg = &averunnable;
  616 
  617         for (i = 0; i < 3; i++)
  618                 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
  619                     nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
  620 
  621         /*
  622          * Schedule the next update to occur after 5 seconds, but add a
  623          * random variation to avoid synchronisation with processes that
  624          * run at regular intervals.
  625          */
  626         callout_reset_sbt(&loadav_callout,
  627             SBT_1US * (4000000 + (int)(random() % 2000001)), SBT_1US,
  628             loadav, NULL, C_DIRECT_EXEC | C_PREL(32));
  629 }
  630 
  631 /* ARGSUSED */
  632 static void
  633 synch_setup(void *dummy)
  634 {
  635         callout_init(&loadav_callout, 1);
  636 
  637         /* Kick off timeout driven events by calling first time. */
  638         loadav(NULL);
  639 }
  640 
  641 int
  642 should_yield(void)
  643 {
  644 
  645         return ((u_int)ticks - (u_int)curthread->td_swvoltick >= hogticks);
  646 }
  647 
  648 void
  649 maybe_yield(void)
  650 {
  651 
  652         if (should_yield())
  653                 kern_yield(PRI_USER);
  654 }
  655 
  656 void
  657 kern_yield(int prio)
  658 {
  659         struct thread *td;
  660 
  661         td = curthread;
  662         DROP_GIANT();
  663         thread_lock(td);
  664         if (prio == PRI_USER)
  665                 prio = td->td_user_pri;
  666         if (prio >= 0)
  667                 sched_prio(td, prio);
  668         mi_switch(SW_VOL | SWT_RELINQUISH);
  669         PICKUP_GIANT();
  670 }
  671 
  672 /*
  673  * General purpose yield system call.
  674  */
  675 int
  676 sys_yield(struct thread *td, struct yield_args *uap)
  677 {
  678 
  679         thread_lock(td);
  680         if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
  681                 sched_prio(td, PRI_MAX_TIMESHARE);
  682         mi_switch(SW_VOL | SWT_RELINQUISH);
  683         td->td_retval[0] = 0;
  684         return (0);
  685 }

Cache object: 8d903ed43c29ff9664d7566fec9fcf64


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.