The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_thread.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
    3  *  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice(s), this list of conditions and the following disclaimer as
   10  *    the first lines of this file unmodified other than the possible
   11  *    addition of one or more copyright notices.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice(s), this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
   17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
   20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   26  * DAMAGE.
   27  */
   28 
   29 #include "opt_witness.h"
   30 #include "opt_kdtrace.h"
   31 #include "opt_hwpmc_hooks.h"
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD$");
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/kernel.h>
   39 #include <sys/lock.h>
   40 #include <sys/mutex.h>
   41 #include <sys/proc.h>
   42 #include <sys/resourcevar.h>
   43 #include <sys/sdt.h>
   44 #include <sys/smp.h>
   45 #include <sys/sched.h>
   46 #include <sys/sleepqueue.h>
   47 #include <sys/selinfo.h>
   48 #include <sys/turnstile.h>
   49 #include <sys/ktr.h>
   50 #include <sys/umtx.h>
   51 #include <sys/cpuset.h>
   52 #ifdef  HWPMC_HOOKS
   53 #include <sys/pmckern.h>
   54 #endif
   55 
   56 #include <security/audit/audit.h>
   57 
   58 #include <vm/vm.h>
   59 #include <vm/vm_extern.h>
   60 #include <vm/uma.h>
   61 #include <sys/eventhandler.h>
   62 
   63 SDT_PROVIDER_DECLARE(proc);
   64 SDT_PROBE_DEFINE(proc, , , lwp_exit, lwp-exit);
   65 
   66 
   67 /*
   68  * thread related storage.
   69  */
   70 static uma_zone_t thread_zone;
   71 
   72 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
   73 static struct mtx zombie_lock;
   74 MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
   75 
   76 static void thread_zombie(struct thread *);
   77 
   78 struct mtx tid_lock;
   79 static struct unrhdr *tid_unrhdr;
   80 
   81 /*
   82  * Prepare a thread for use.
   83  */
   84 static int
   85 thread_ctor(void *mem, int size, void *arg, int flags)
   86 {
   87         struct thread   *td;
   88 
   89         td = (struct thread *)mem;
   90         td->td_state = TDS_INACTIVE;
   91         td->td_oncpu = NOCPU;
   92 
   93         td->td_tid = alloc_unr(tid_unrhdr);
   94         td->td_syscalls = 0;
   95 
   96         /*
   97          * Note that td_critnest begins life as 1 because the thread is not
   98          * running and is thereby implicitly waiting to be on the receiving
   99          * end of a context switch.
  100          */
  101         td->td_critnest = 1;
  102         EVENTHANDLER_INVOKE(thread_ctor, td);
  103 #ifdef AUDIT
  104         audit_thread_alloc(td);
  105 #endif
  106         umtx_thread_alloc(td);
  107         return (0);
  108 }
  109 
  110 /*
  111  * Reclaim a thread after use.
  112  */
  113 static void
  114 thread_dtor(void *mem, int size, void *arg)
  115 {
  116         struct thread *td;
  117 
  118         td = (struct thread *)mem;
  119 
  120 #ifdef INVARIANTS
  121         /* Verify that this thread is in a safe state to free. */
  122         switch (td->td_state) {
  123         case TDS_INHIBITED:
  124         case TDS_RUNNING:
  125         case TDS_CAN_RUN:
  126         case TDS_RUNQ:
  127                 /*
  128                  * We must never unlink a thread that is in one of
  129                  * these states, because it is currently active.
  130                  */
  131                 panic("bad state for thread unlinking");
  132                 /* NOTREACHED */
  133         case TDS_INACTIVE:
  134                 break;
  135         default:
  136                 panic("bad thread state");
  137                 /* NOTREACHED */
  138         }
  139 #endif
  140 #ifdef AUDIT
  141         audit_thread_free(td);
  142 #endif
  143         /* Free all OSD associated to this thread. */
  144         osd_thread_exit(td);
  145 
  146         EVENTHANDLER_INVOKE(thread_dtor, td);
  147         free_unr(tid_unrhdr, td->td_tid);
  148 }
  149 
  150 /*
  151  * Initialize type-stable parts of a thread (when newly created).
  152  */
  153 static int
  154 thread_init(void *mem, int size, int flags)
  155 {
  156         struct thread *td;
  157 
  158         td = (struct thread *)mem;
  159 
  160         td->td_sleepqueue = sleepq_alloc();
  161         td->td_turnstile = turnstile_alloc();
  162         td->td_vp_reserv = 0;
  163         EVENTHANDLER_INVOKE(thread_init, td);
  164         td->td_sched = (struct td_sched *)&td[1];
  165         umtx_thread_init(td);
  166         td->td_kstack = 0;
  167         return (0);
  168 }
  169 
  170 /*
  171  * Tear down type-stable parts of a thread (just before being discarded).
  172  */
  173 static void
  174 thread_fini(void *mem, int size)
  175 {
  176         struct thread *td;
  177 
  178         td = (struct thread *)mem;
  179         EVENTHANDLER_INVOKE(thread_fini, td);
  180         turnstile_free(td->td_turnstile);
  181         sleepq_free(td->td_sleepqueue);
  182         umtx_thread_fini(td);
  183         seltdfini(td);
  184 }
  185 
  186 /*
  187  * For a newly created process,
  188  * link up all the structures and its initial threads etc.
  189  * called from:
  190  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
  191  * proc_dtor() (should go away)
  192  * proc_init()
  193  */
  194 void
  195 proc_linkup0(struct proc *p, struct thread *td)
  196 {
  197         TAILQ_INIT(&p->p_threads);           /* all threads in proc */
  198         proc_linkup(p, td);
  199 }
  200 
  201 void
  202 proc_linkup(struct proc *p, struct thread *td)
  203 {
  204 
  205         sigqueue_init(&p->p_sigqueue, p);
  206         p->p_ksi = ksiginfo_alloc(1);
  207         if (p->p_ksi != NULL) {
  208                 /* XXX p_ksi may be null if ksiginfo zone is not ready */
  209                 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
  210         }
  211         LIST_INIT(&p->p_mqnotifier);
  212         p->p_numthreads = 0;
  213         thread_link(td, p);
  214 }
  215 
  216 /*
  217  * Initialize global thread allocation resources.
  218  */
  219 void
  220 threadinit(void)
  221 {
  222 
  223         mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
  224         /* leave one number for thread0 */
  225         tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock);
  226 
  227         thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
  228             thread_ctor, thread_dtor, thread_init, thread_fini,
  229             16 - 1, 0);
  230 }
  231 
  232 /*
  233  * Place an unused thread on the zombie list.
  234  * Use the slpq as that must be unused by now.
  235  */
  236 void
  237 thread_zombie(struct thread *td)
  238 {
  239         mtx_lock_spin(&zombie_lock);
  240         TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
  241         mtx_unlock_spin(&zombie_lock);
  242 }
  243 
  244 /*
  245  * Release a thread that has exited after cpu_throw().
  246  */
  247 void
  248 thread_stash(struct thread *td)
  249 {
  250         atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
  251         thread_zombie(td);
  252 }
  253 
  254 /*
  255  * Reap zombie resources.
  256  */
  257 void
  258 thread_reap(void)
  259 {
  260         struct thread *td_first, *td_next;
  261 
  262         /*
  263          * Don't even bother to lock if none at this instant,
  264          * we really don't care about the next instant..
  265          */
  266         if (!TAILQ_EMPTY(&zombie_threads)) {
  267                 mtx_lock_spin(&zombie_lock);
  268                 td_first = TAILQ_FIRST(&zombie_threads);
  269                 if (td_first)
  270                         TAILQ_INIT(&zombie_threads);
  271                 mtx_unlock_spin(&zombie_lock);
  272                 while (td_first) {
  273                         td_next = TAILQ_NEXT(td_first, td_slpq);
  274                         if (td_first->td_ucred)
  275                                 crfree(td_first->td_ucred);
  276                         thread_free(td_first);
  277                         td_first = td_next;
  278                 }
  279         }
  280 }
  281 
  282 /*
  283  * Allocate a thread.
  284  */
  285 struct thread *
  286 thread_alloc(int pages)
  287 {
  288         struct thread *td;
  289 
  290         thread_reap(); /* check if any zombies to get */
  291 
  292         td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
  293         KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
  294         if (!vm_thread_new(td, pages)) {
  295                 uma_zfree(thread_zone, td);
  296                 return (NULL);
  297         }
  298         cpu_thread_alloc(td);
  299         return (td);
  300 }
  301 
  302 int
  303 thread_alloc_stack(struct thread *td, int pages)
  304 {
  305 
  306         KASSERT(td->td_kstack == 0,
  307             ("thread_alloc_stack called on a thread with kstack"));
  308         if (!vm_thread_new(td, pages))
  309                 return (0);
  310         cpu_thread_alloc(td);
  311         return (1);
  312 }
  313 
  314 /*
  315  * Deallocate a thread.
  316  */
  317 void
  318 thread_free(struct thread *td)
  319 {
  320 
  321         lock_profile_thread_exit(td);
  322         if (td->td_cpuset)
  323                 cpuset_rel(td->td_cpuset);
  324         td->td_cpuset = NULL;
  325         cpu_thread_free(td);
  326         if (td->td_kstack != 0)
  327                 vm_thread_dispose(td);
  328         uma_zfree(thread_zone, td);
  329 }
  330 
  331 /*
  332  * Discard the current thread and exit from its context.
  333  * Always called with scheduler locked.
  334  *
  335  * Because we can't free a thread while we're operating under its context,
  336  * push the current thread into our CPU's deadthread holder. This means
  337  * we needn't worry about someone else grabbing our context before we
  338  * do a cpu_throw().
  339  */
  340 void
  341 thread_exit(void)
  342 {
  343         uint64_t runtime, new_switchtime;
  344         struct thread *td;
  345         struct thread *td2;
  346         struct proc *p;
  347         int wakeup_swapper;
  348 
  349         td = curthread;
  350         p = td->td_proc;
  351 
  352         PROC_SLOCK_ASSERT(p, MA_OWNED);
  353         mtx_assert(&Giant, MA_NOTOWNED);
  354 
  355         PROC_LOCK_ASSERT(p, MA_OWNED);
  356         KASSERT(p != NULL, ("thread exiting without a process"));
  357         CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
  358             (long)p->p_pid, td->td_name);
  359         KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
  360 
  361 #ifdef AUDIT
  362         AUDIT_SYSCALL_EXIT(0, td);
  363 #endif
  364         umtx_thread_exit(td);
  365         /*
  366          * drop FPU & debug register state storage, or any other
  367          * architecture specific resources that
  368          * would not be on a new untouched process.
  369          */
  370         cpu_thread_exit(td);    /* XXXSMP */
  371 
  372         /*
  373          * The last thread is left attached to the process
  374          * So that the whole bundle gets recycled. Skip
  375          * all this stuff if we never had threads.
  376          * EXIT clears all sign of other threads when
  377          * it goes to single threading, so the last thread always
  378          * takes the short path.
  379          */
  380         if (p->p_flag & P_HADTHREADS) {
  381                 if (p->p_numthreads > 1) {
  382                         thread_unlink(td);
  383                         td2 = FIRST_THREAD_IN_PROC(p);
  384                         sched_exit_thread(td2, td);
  385 
  386                         /*
  387                          * The test below is NOT true if we are the
  388                          * sole exiting thread. P_STOPPED_SINGLE is unset
  389                          * in exit1() after it is the only survivor.
  390                          */
  391                         if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  392                                 if (p->p_numthreads == p->p_suspcount) {
  393                                         thread_lock(p->p_singlethread);
  394                                         wakeup_swapper = thread_unsuspend_one(
  395                                                 p->p_singlethread);
  396                                         thread_unlock(p->p_singlethread);
  397                                         if (wakeup_swapper)
  398                                                 kick_proc0();
  399                                 }
  400                         }
  401 
  402                         atomic_add_int(&td->td_proc->p_exitthreads, 1);
  403                         PCPU_SET(deadthread, td);
  404                 } else {
  405                         /*
  406                          * The last thread is exiting.. but not through exit()
  407                          */
  408                         panic ("thread_exit: Last thread exiting on its own");
  409                 }
  410         } 
  411 #ifdef  HWPMC_HOOKS
  412         /*
  413          * If this thread is part of a process that is being tracked by hwpmc(4),
  414          * inform the module of the thread's impending exit.
  415          */
  416         if (PMC_PROC_IS_USING_PMCS(td->td_proc))
  417                 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
  418 #endif
  419         PROC_UNLOCK(p);
  420 
  421         /* Do the same timestamp bookkeeping that mi_switch() would do. */
  422         new_switchtime = cpu_ticks();
  423         runtime = new_switchtime - PCPU_GET(switchtime);
  424         td->td_runtime += runtime;
  425         td->td_incruntime += runtime;
  426         PCPU_SET(switchtime, new_switchtime);
  427         PCPU_SET(switchticks, ticks);
  428         PCPU_INC(cnt.v_swtch);
  429 
  430         /* Save our resource usage in our process. */
  431         td->td_ru.ru_nvcsw++;
  432         ruxagg(p, td);
  433         rucollect(&p->p_ru, &td->td_ru);
  434 
  435         thread_lock(td);
  436         PROC_SUNLOCK(p);
  437         td->td_state = TDS_INACTIVE;
  438 #ifdef WITNESS
  439         witness_thread_exit(td);
  440 #endif
  441         CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
  442         sched_throw(td);
  443         panic("I'm a teapot!");
  444         /* NOTREACHED */
  445 }
  446 
  447 /*
  448  * Do any thread specific cleanups that may be needed in wait()
  449  * called with Giant, proc and schedlock not held.
  450  */
  451 void
  452 thread_wait(struct proc *p)
  453 {
  454         struct thread *td;
  455 
  456         mtx_assert(&Giant, MA_NOTOWNED);
  457         KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
  458         td = FIRST_THREAD_IN_PROC(p);
  459         /* Lock the last thread so we spin until it exits cpu_throw(). */
  460         thread_lock(td);
  461         thread_unlock(td);
  462         /* Wait for any remaining threads to exit cpu_throw(). */
  463         while (p->p_exitthreads)
  464                 sched_relinquish(curthread);
  465         lock_profile_thread_exit(td);
  466         cpuset_rel(td->td_cpuset);
  467         td->td_cpuset = NULL;
  468         cpu_thread_clean(td);
  469         crfree(td->td_ucred);
  470         thread_reap();  /* check for zombie threads etc. */
  471 }
  472 
  473 /*
  474  * Link a thread to a process.
  475  * set up anything that needs to be initialized for it to
  476  * be used by the process.
  477  */
  478 void
  479 thread_link(struct thread *td, struct proc *p)
  480 {
  481 
  482         /*
  483          * XXX This can't be enabled because it's called for proc0 before
  484          * its lock has been created.
  485          * PROC_LOCK_ASSERT(p, MA_OWNED);
  486          */
  487         td->td_state    = TDS_INACTIVE;
  488         td->td_proc     = p;
  489         td->td_flags    = TDF_INMEM;
  490 
  491         LIST_INIT(&td->td_contested);
  492         LIST_INIT(&td->td_lprof[0]);
  493         LIST_INIT(&td->td_lprof[1]);
  494         sigqueue_init(&td->td_sigqueue, p);
  495         callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
  496         TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
  497         p->p_numthreads++;
  498 }
  499 
  500 /*
  501  * Convert a process with one thread to an unthreaded process.
  502  */
  503 void
  504 thread_unthread(struct thread *td)
  505 {
  506         struct proc *p = td->td_proc;
  507 
  508         KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
  509         p->p_flag &= ~P_HADTHREADS;
  510 }
  511 
  512 /*
  513  * Called from:
  514  *  thread_exit()
  515  */
  516 void
  517 thread_unlink(struct thread *td)
  518 {
  519         struct proc *p = td->td_proc;
  520 
  521         PROC_LOCK_ASSERT(p, MA_OWNED);
  522         TAILQ_REMOVE(&p->p_threads, td, td_plist);
  523         p->p_numthreads--;
  524         /* could clear a few other things here */
  525         /* Must  NOT clear links to proc! */
  526 }
  527 
  528 static int
  529 calc_remaining(struct proc *p, int mode)
  530 {
  531         int remaining;
  532 
  533         PROC_LOCK_ASSERT(p, MA_OWNED);
  534         PROC_SLOCK_ASSERT(p, MA_OWNED);
  535         if (mode == SINGLE_EXIT)
  536                 remaining = p->p_numthreads;
  537         else if (mode == SINGLE_BOUNDARY)
  538                 remaining = p->p_numthreads - p->p_boundary_count;
  539         else if (mode == SINGLE_NO_EXIT)
  540                 remaining = p->p_numthreads - p->p_suspcount;
  541         else
  542                 panic("calc_remaining: wrong mode %d", mode);
  543         return (remaining);
  544 }
  545 
  546 /*
  547  * Enforce single-threading.
  548  *
  549  * Returns 1 if the caller must abort (another thread is waiting to
  550  * exit the process or similar). Process is locked!
  551  * Returns 0 when you are successfully the only thread running.
  552  * A process has successfully single threaded in the suspend mode when
  553  * There are no threads in user mode. Threads in the kernel must be
  554  * allowed to continue until they get to the user boundary. They may even
  555  * copy out their return values and data before suspending. They may however be
  556  * accelerated in reaching the user boundary as we will wake up
  557  * any sleeping threads that are interruptable. (PCATCH).
  558  */
  559 int
  560 thread_single(int mode)
  561 {
  562         struct thread *td;
  563         struct thread *td2;
  564         struct proc *p;
  565         int remaining, wakeup_swapper;
  566 
  567         td = curthread;
  568         p = td->td_proc;
  569         mtx_assert(&Giant, MA_NOTOWNED);
  570         PROC_LOCK_ASSERT(p, MA_OWNED);
  571         KASSERT((td != NULL), ("curthread is NULL"));
  572 
  573         if ((p->p_flag & P_HADTHREADS) == 0)
  574                 return (0);
  575 
  576         /* Is someone already single threading? */
  577         if (p->p_singlethread != NULL && p->p_singlethread != td)
  578                 return (1);
  579 
  580         if (mode == SINGLE_EXIT) {
  581                 p->p_flag |= P_SINGLE_EXIT;
  582                 p->p_flag &= ~P_SINGLE_BOUNDARY;
  583         } else {
  584                 p->p_flag &= ~P_SINGLE_EXIT;
  585                 if (mode == SINGLE_BOUNDARY)
  586                         p->p_flag |= P_SINGLE_BOUNDARY;
  587                 else
  588                         p->p_flag &= ~P_SINGLE_BOUNDARY;
  589         }
  590         p->p_flag |= P_STOPPED_SINGLE;
  591         PROC_SLOCK(p);
  592         p->p_singlethread = td;
  593         remaining = calc_remaining(p, mode);
  594         while (remaining != 1) {
  595                 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
  596                         goto stopme;
  597                 wakeup_swapper = 0;
  598                 FOREACH_THREAD_IN_PROC(p, td2) {
  599                         if (td2 == td)
  600                                 continue;
  601                         thread_lock(td2);
  602                         td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
  603                         if (TD_IS_INHIBITED(td2)) {
  604                                 switch (mode) {
  605                                 case SINGLE_EXIT:
  606                                         if (TD_IS_SUSPENDED(td2))
  607                                                 wakeup_swapper |=
  608                                                     thread_unsuspend_one(td2);
  609                                         if (TD_ON_SLEEPQ(td2) &&
  610                                             (td2->td_flags & TDF_SINTR))
  611                                                 wakeup_swapper |=
  612                                                     sleepq_abort(td2, EINTR);
  613                                         break;
  614                                 case SINGLE_BOUNDARY:
  615                                         if (TD_IS_SUSPENDED(td2) &&
  616                                             !(td2->td_flags & TDF_BOUNDARY))
  617                                                 wakeup_swapper |=
  618                                                     thread_unsuspend_one(td2);
  619                                         if (TD_ON_SLEEPQ(td2) &&
  620                                             (td2->td_flags & TDF_SINTR))
  621                                                 wakeup_swapper |=
  622                                                     sleepq_abort(td2, ERESTART);
  623                                         break;
  624                                 case SINGLE_NO_EXIT:
  625                                         if (TD_IS_SUSPENDED(td2) &&
  626                                             !(td2->td_flags & TDF_BOUNDARY))
  627                                                 wakeup_swapper |=
  628                                                     thread_unsuspend_one(td2);
  629                                         if (TD_ON_SLEEPQ(td2) &&
  630                                             (td2->td_flags & TDF_SINTR))
  631                                                 wakeup_swapper |=
  632                                                     sleepq_abort(td2, ERESTART);
  633                                         break;
  634                                 default:
  635                                         break;
  636                                 }
  637                         }
  638 #ifdef SMP
  639                         else if (TD_IS_RUNNING(td2) && td != td2) {
  640                                 forward_signal(td2);
  641                         }
  642 #endif
  643                         thread_unlock(td2);
  644                 }
  645                 if (wakeup_swapper)
  646                         kick_proc0();
  647                 remaining = calc_remaining(p, mode);
  648 
  649                 /*
  650                  * Maybe we suspended some threads.. was it enough?
  651                  */
  652                 if (remaining == 1)
  653                         break;
  654 
  655 stopme:
  656                 /*
  657                  * Wake us up when everyone else has suspended.
  658                  * In the mean time we suspend as well.
  659                  */
  660                 thread_suspend_switch(td);
  661                 remaining = calc_remaining(p, mode);
  662         }
  663         if (mode == SINGLE_EXIT) {
  664                 /*
  665                  * We have gotten rid of all the other threads and we
  666                  * are about to either exit or exec. In either case,
  667                  * we try our utmost  to revert to being a non-threaded
  668                  * process.
  669                  */
  670                 p->p_singlethread = NULL;
  671                 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
  672                 thread_unthread(td);
  673         }
  674         PROC_SUNLOCK(p);
  675         return (0);
  676 }
  677 
  678 /*
  679  * Called in from locations that can safely check to see
  680  * whether we have to suspend or at least throttle for a
  681  * single-thread event (e.g. fork).
  682  *
  683  * Such locations include userret().
  684  * If the "return_instead" argument is non zero, the thread must be able to
  685  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
  686  *
  687  * The 'return_instead' argument tells the function if it may do a
  688  * thread_exit() or suspend, or whether the caller must abort and back
  689  * out instead.
  690  *
  691  * If the thread that set the single_threading request has set the
  692  * P_SINGLE_EXIT bit in the process flags then this call will never return
  693  * if 'return_instead' is false, but will exit.
  694  *
  695  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
  696  *---------------+--------------------+---------------------
  697  *       0       | returns 0          |   returns 0 or 1
  698  *               | when ST ends       |   immediatly
  699  *---------------+--------------------+---------------------
  700  *       1       | thread exits       |   returns 1
  701  *               |                    |  immediatly
  702  * 0 = thread_exit() or suspension ok,
  703  * other = return error instead of stopping the thread.
  704  *
  705  * While a full suspension is under effect, even a single threading
  706  * thread would be suspended if it made this call (but it shouldn't).
  707  * This call should only be made from places where
  708  * thread_exit() would be safe as that may be the outcome unless
  709  * return_instead is set.
  710  */
  711 int
  712 thread_suspend_check(int return_instead)
  713 {
  714         struct thread *td;
  715         struct proc *p;
  716         int wakeup_swapper;
  717 
  718         td = curthread;
  719         p = td->td_proc;
  720         mtx_assert(&Giant, MA_NOTOWNED);
  721         PROC_LOCK_ASSERT(p, MA_OWNED);
  722         while (P_SHOULDSTOP(p) ||
  723               ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) {
  724                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  725                         KASSERT(p->p_singlethread != NULL,
  726                             ("singlethread not set"));
  727                         /*
  728                          * The only suspension in action is a
  729                          * single-threading. Single threader need not stop.
  730                          * XXX Should be safe to access unlocked
  731                          * as it can only be set to be true by us.
  732                          */
  733                         if (p->p_singlethread == td)
  734                                 return (0);     /* Exempt from stopping. */
  735                 }
  736                 if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
  737                         return (EINTR);
  738 
  739                 /* Should we goto user boundary if we didn't come from there? */
  740                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
  741                     (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
  742                         return (ERESTART);
  743 
  744                 /*
  745                  * Ignore suspend requests for stop signals if they
  746                  * are deferred.
  747                  */
  748                 if (P_SHOULDSTOP(p) == P_STOPPED_SIG &&
  749                     td->td_flags & TDF_SBDRY) {
  750                         KASSERT(return_instead,
  751                             ("TDF_SBDRY set for unsafe thread_suspend_check"));
  752                         return (0);
  753                 }
  754 
  755                 /* If thread will exit, flush its pending signals */
  756                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
  757                         sigqueue_flush(&td->td_sigqueue);
  758 
  759                 PROC_SLOCK(p);
  760                 thread_stopped(p);
  761                 /*
  762                  * If the process is waiting for us to exit,
  763                  * this thread should just suicide.
  764                  * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
  765                  */
  766                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
  767                         thread_exit();
  768                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  769                         if (p->p_numthreads == p->p_suspcount + 1) {
  770                                 thread_lock(p->p_singlethread);
  771                                 wakeup_swapper =
  772                                     thread_unsuspend_one(p->p_singlethread);
  773                                 thread_unlock(p->p_singlethread);
  774                                 if (wakeup_swapper)
  775                                         kick_proc0();
  776                         }
  777                 }
  778                 PROC_UNLOCK(p);
  779                 thread_lock(td);
  780                 /*
  781                  * When a thread suspends, it just
  782                  * gets taken off all queues.
  783                  */
  784                 thread_suspend_one(td);
  785                 if (return_instead == 0) {
  786                         p->p_boundary_count++;
  787                         td->td_flags |= TDF_BOUNDARY;
  788                 }
  789                 PROC_SUNLOCK(p);
  790                 mi_switch(SW_INVOL | SWT_SUSPEND, NULL);
  791                 if (return_instead == 0)
  792                         td->td_flags &= ~TDF_BOUNDARY;
  793                 thread_unlock(td);
  794                 PROC_LOCK(p);
  795                 if (return_instead == 0) {
  796                         PROC_SLOCK(p);
  797                         p->p_boundary_count--;
  798                         PROC_SUNLOCK(p);
  799                 }
  800         }
  801         return (0);
  802 }
  803 
  804 void
  805 thread_suspend_switch(struct thread *td)
  806 {
  807         struct proc *p;
  808 
  809         p = td->td_proc;
  810         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
  811         PROC_LOCK_ASSERT(p, MA_OWNED);
  812         PROC_SLOCK_ASSERT(p, MA_OWNED);
  813         /*
  814          * We implement thread_suspend_one in stages here to avoid
  815          * dropping the proc lock while the thread lock is owned.
  816          */
  817         thread_stopped(p);
  818         p->p_suspcount++;
  819         PROC_UNLOCK(p);
  820         thread_lock(td);
  821         td->td_flags &= ~TDF_NEEDSUSPCHK;
  822         TD_SET_SUSPENDED(td);
  823         sched_sleep(td, 0);
  824         PROC_SUNLOCK(p);
  825         DROP_GIANT();
  826         mi_switch(SW_VOL | SWT_SUSPEND, NULL);
  827         thread_unlock(td);
  828         PICKUP_GIANT();
  829         PROC_LOCK(p);
  830         PROC_SLOCK(p);
  831 }
  832 
  833 void
  834 thread_suspend_one(struct thread *td)
  835 {
  836         struct proc *p = td->td_proc;
  837 
  838         PROC_SLOCK_ASSERT(p, MA_OWNED);
  839         THREAD_LOCK_ASSERT(td, MA_OWNED);
  840         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
  841         p->p_suspcount++;
  842         td->td_flags &= ~TDF_NEEDSUSPCHK;
  843         TD_SET_SUSPENDED(td);
  844         sched_sleep(td, 0);
  845 }
  846 
  847 int
  848 thread_unsuspend_one(struct thread *td)
  849 {
  850         struct proc *p = td->td_proc;
  851 
  852         PROC_SLOCK_ASSERT(p, MA_OWNED);
  853         THREAD_LOCK_ASSERT(td, MA_OWNED);
  854         KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
  855         TD_CLR_SUSPENDED(td);
  856         p->p_suspcount--;
  857         return (setrunnable(td));
  858 }
  859 
  860 /*
  861  * Allow all threads blocked by single threading to continue running.
  862  */
  863 void
  864 thread_unsuspend(struct proc *p)
  865 {
  866         struct thread *td;
  867         int wakeup_swapper;
  868 
  869         PROC_LOCK_ASSERT(p, MA_OWNED);
  870         PROC_SLOCK_ASSERT(p, MA_OWNED);
  871         wakeup_swapper = 0;
  872         if (!P_SHOULDSTOP(p)) {
  873                 FOREACH_THREAD_IN_PROC(p, td) {
  874                         thread_lock(td);
  875                         if (TD_IS_SUSPENDED(td)) {
  876                                 wakeup_swapper |= thread_unsuspend_one(td);
  877                         }
  878                         thread_unlock(td);
  879                 }
  880         } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
  881             (p->p_numthreads == p->p_suspcount)) {
  882                 /*
  883                  * Stopping everything also did the job for the single
  884                  * threading request. Now we've downgraded to single-threaded,
  885                  * let it continue.
  886                  */
  887                 thread_lock(p->p_singlethread);
  888                 wakeup_swapper = thread_unsuspend_one(p->p_singlethread);
  889                 thread_unlock(p->p_singlethread);
  890         }
  891         if (wakeup_swapper)
  892                 kick_proc0();
  893 }
  894 
  895 /*
  896  * End the single threading mode..
  897  */
  898 void
  899 thread_single_end(void)
  900 {
  901         struct thread *td;
  902         struct proc *p;
  903         int wakeup_swapper;
  904 
  905         td = curthread;
  906         p = td->td_proc;
  907         PROC_LOCK_ASSERT(p, MA_OWNED);
  908         p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
  909         PROC_SLOCK(p);
  910         p->p_singlethread = NULL;
  911         wakeup_swapper = 0;
  912         /*
  913          * If there are other threads they may now run,
  914          * unless of course there is a blanket 'stop order'
  915          * on the process. The single threader must be allowed
  916          * to continue however as this is a bad place to stop.
  917          */
  918         if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
  919                 FOREACH_THREAD_IN_PROC(p, td) {
  920                         thread_lock(td);
  921                         if (TD_IS_SUSPENDED(td)) {
  922                                 wakeup_swapper |= thread_unsuspend_one(td);
  923                         }
  924                         thread_unlock(td);
  925                 }
  926         }
  927         PROC_SUNLOCK(p);
  928         if (wakeup_swapper)
  929                 kick_proc0();
  930 }
  931 
  932 struct thread *
  933 thread_find(struct proc *p, lwpid_t tid)
  934 {
  935         struct thread *td;
  936 
  937         PROC_LOCK_ASSERT(p, MA_OWNED);
  938         FOREACH_THREAD_IN_PROC(p, td) {
  939                 if (td->td_tid == tid)
  940                         break;
  941         }
  942         return (td);
  943 }

Cache object: 4cc96c0b88ea30a5808b925d21b42fb6


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.