kern_thread.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
    3  *  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice(s), this list of conditions and the following disclaimer as
   10  *    the first lines of this file unmodified other than the possible
   11  *    addition of one or more copyright notices.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice(s), this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
   17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
   20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   26  * DAMAGE.
   27  */
   28 
   29 #include "opt_witness.h"
   30 #include "opt_hwpmc_hooks.h"
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD: releng/9.0/sys/kern/kern_thread.c 227886 2011-11-23 15:16:05Z kib $");
   34 
   35 #include <sys/param.h>
   36 #include <sys/systm.h>
   37 #include <sys/kernel.h>
   38 #include <sys/lock.h>
   39 #include <sys/mutex.h>
   40 #include <sys/proc.h>
   41 #include <sys/resourcevar.h>
   42 #include <sys/smp.h>
   43 #include <sys/sched.h>
   44 #include <sys/sleepqueue.h>
   45 #include <sys/selinfo.h>
   46 #include <sys/turnstile.h>
   47 #include <sys/ktr.h>
   48 #include <sys/rwlock.h>
   49 #include <sys/umtx.h>
   50 #include <sys/cpuset.h>
   51 #ifdef  HWPMC_HOOKS
   52 #include <sys/pmckern.h>
   53 #endif
   54 
   55 #include <security/audit/audit.h>
   56 
   57 #include <vm/vm.h>
   58 #include <vm/vm_extern.h>
   59 #include <vm/uma.h>
   60 #include <sys/eventhandler.h>
   61 
   62 /*
   63  * thread related storage.
   64  */
   65 static uma_zone_t thread_zone;
   66 
   67 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
   68 static struct mtx zombie_lock;
   69 MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
   70 
   71 static void thread_zombie(struct thread *);
   72 
   73 #define TID_BUFFER_SIZE 1024
   74 
   75 struct mtx tid_lock;
   76 static struct unrhdr *tid_unrhdr;
   77 static lwpid_t tid_buffer[TID_BUFFER_SIZE];
   78 static int tid_head, tid_tail;
   79 static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
   80 
   81 struct  tidhashhead *tidhashtbl;
   82 u_long  tidhash;
   83 struct  rwlock tidhash_lock;
   84 
   85 static lwpid_t
   86 tid_alloc(void)
   87 {
   88         lwpid_t tid;
   89 
   90         tid = alloc_unr(tid_unrhdr);
   91         if (tid != -1)
   92                 return (tid);
   93         mtx_lock(&tid_lock);
   94         if (tid_head == tid_tail) {
   95                 mtx_unlock(&tid_lock);
   96                 return (-1);
   97         }
   98         tid = tid_buffer[tid_head++];
   99         tid_head %= TID_BUFFER_SIZE;
  100         mtx_unlock(&tid_lock);
  101         return (tid);
  102 }
  103 
  104 static void
  105 tid_free(lwpid_t tid)
  106 {
  107         lwpid_t tmp_tid = -1;
  108 
  109         mtx_lock(&tid_lock);
  110         if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) {
  111                 tmp_tid = tid_buffer[tid_head++];
  112                 tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
  113         }
  114         tid_buffer[tid_tail++] = tid;
  115         tid_tail %= TID_BUFFER_SIZE;
  116         mtx_unlock(&tid_lock);
  117         if (tmp_tid != -1)
  118                 free_unr(tid_unrhdr, tmp_tid);
  119 }
  120 
  121 /*
  122  * Prepare a thread for use.
  123  */
  124 static int
  125 thread_ctor(void *mem, int size, void *arg, int flags)
  126 {
  127         struct thread   *td;
  128 
  129         td = (struct thread *)mem;
  130         td->td_state = TDS_INACTIVE;
  131         td->td_oncpu = NOCPU;
  132 
  133         td->td_tid = tid_alloc();
  134 
  135         /*
  136          * Note that td_critnest begins life as 1 because the thread is not
  137          * running and is thereby implicitly waiting to be on the receiving
  138          * end of a context switch.
  139          */
  140         td->td_critnest = 1;
  141         td->td_lend_user_pri = PRI_MAX;
  142         EVENTHANDLER_INVOKE(thread_ctor, td);
  143 #ifdef AUDIT
  144         audit_thread_alloc(td);
  145 #endif
  146         umtx_thread_alloc(td);
  147         return (0);
  148 }
  149 
  150 /*
  151  * Reclaim a thread after use.
  152  */
  153 static void
  154 thread_dtor(void *mem, int size, void *arg)
  155 {
  156         struct thread *td;
  157 
  158         td = (struct thread *)mem;
  159 
  160 #ifdef INVARIANTS
  161         /* Verify that this thread is in a safe state to free. */
  162         switch (td->td_state) {
  163         case TDS_INHIBITED:
  164         case TDS_RUNNING:
  165         case TDS_CAN_RUN:
  166         case TDS_RUNQ:
  167                 /*
  168                  * We must never unlink a thread that is in one of
  169                  * these states, because it is currently active.
  170                  */
  171                 panic("bad state for thread unlinking");
  172                 /* NOTREACHED */
  173         case TDS_INACTIVE:
  174                 break;
  175         default:
  176                 panic("bad thread state");
  177                 /* NOTREACHED */
  178         }
  179 #endif
  180 #ifdef AUDIT
  181         audit_thread_free(td);
  182 #endif
  183         /* Free all OSD associated to this thread. */
  184         osd_thread_exit(td);
  185 
  186         EVENTHANDLER_INVOKE(thread_dtor, td);
  187         tid_free(td->td_tid);
  188 }
  189 
  190 /*
  191  * Initialize type-stable parts of a thread (when newly created).
  192  */
  193 static int
  194 thread_init(void *mem, int size, int flags)
  195 {
  196         struct thread *td;
  197 
  198         td = (struct thread *)mem;
  199 
  200         td->td_sleepqueue = sleepq_alloc();
  201         td->td_turnstile = turnstile_alloc();
  202         EVENTHANDLER_INVOKE(thread_init, td);
  203         td->td_sched = (struct td_sched *)&td[1];
  204         umtx_thread_init(td);
  205         td->td_kstack = 0;
  206         return (0);
  207 }
  208 
  209 /*
  210  * Tear down type-stable parts of a thread (just before being discarded).
  211  */
  212 static void
  213 thread_fini(void *mem, int size)
  214 {
  215         struct thread *td;
  216 
  217         td = (struct thread *)mem;
  218         EVENTHANDLER_INVOKE(thread_fini, td);
  219         turnstile_free(td->td_turnstile);
  220         sleepq_free(td->td_sleepqueue);
  221         umtx_thread_fini(td);
  222         seltdfini(td);
  223 }
  224 
  225 /*
  226  * For a newly created process,
  227  * link up all the structures and its initial threads etc.
  228  * called from:
  229  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
  230  * proc_dtor() (should go away)
  231  * proc_init()
  232  */
  233 void
  234 proc_linkup0(struct proc *p, struct thread *td)
  235 {
  236         TAILQ_INIT(&p->p_threads);           /* all threads in proc */
  237         proc_linkup(p, td);
  238 }
  239 
  240 void
  241 proc_linkup(struct proc *p, struct thread *td)
  242 {
  243 
  244         sigqueue_init(&p->p_sigqueue, p);
  245         p->p_ksi = ksiginfo_alloc(1);
  246         if (p->p_ksi != NULL) {
  247                 /* XXX p_ksi may be null if ksiginfo zone is not ready */
  248                 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
  249         }
  250         LIST_INIT(&p->p_mqnotifier);
  251         p->p_numthreads = 0;
  252         thread_link(td, p);
  253 }
  254 
  255 /*
  256  * Initialize global thread allocation resources.
  257  */
  258 void
  259 threadinit(void)
  260 {
  261 
  262         mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
  263         /* leave one number for thread0 */
  264         tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock);
  265 
  266         thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
  267             thread_ctor, thread_dtor, thread_init, thread_fini,
  268             16 - 1, 0);
  269         tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
  270         rw_init(&tidhash_lock, "tidhash");
  271 }
  272 
  273 /*
  274  * Place an unused thread on the zombie list.
  275  * Use the slpq as that must be unused by now.
  276  */
  277 void
  278 thread_zombie(struct thread *td)
  279 {
  280         mtx_lock_spin(&zombie_lock);
  281         TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
  282         mtx_unlock_spin(&zombie_lock);
  283 }
  284 
  285 /*
  286  * Release a thread that has exited after cpu_throw().
  287  */
  288 void
  289 thread_stash(struct thread *td)
  290 {
  291         atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
  292         thread_zombie(td);
  293 }
  294 
  295 /*
  296  * Reap zombie resources.
  297  */
  298 void
  299 thread_reap(void)
  300 {
  301         struct thread *td_first, *td_next;
  302 
  303         /*
  304          * Don't even bother to lock if none at this instant,
  305          * we really don't care about the next instant..
  306          */
  307         if (!TAILQ_EMPTY(&zombie_threads)) {
  308                 mtx_lock_spin(&zombie_lock);
  309                 td_first = TAILQ_FIRST(&zombie_threads);
  310                 if (td_first)
  311                         TAILQ_INIT(&zombie_threads);
  312                 mtx_unlock_spin(&zombie_lock);
  313                 while (td_first) {
  314                         td_next = TAILQ_NEXT(td_first, td_slpq);
  315                         if (td_first->td_ucred)
  316                                 crfree(td_first->td_ucred);
  317                         thread_free(td_first);
  318                         td_first = td_next;
  319                 }
  320         }
  321 }
  322 
  323 /*
  324  * Allocate a thread.
  325  */
  326 struct thread *
  327 thread_alloc(int pages)
  328 {
  329         struct thread *td;
  330 
  331         thread_reap(); /* check if any zombies to get */
  332 
  333         td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
  334         KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
  335         if (!vm_thread_new(td, pages)) {
  336                 uma_zfree(thread_zone, td);
  337                 return (NULL);
  338         }
  339         cpu_thread_alloc(td);
  340         return (td);
  341 }
  342 
  343 int
  344 thread_alloc_stack(struct thread *td, int pages)
  345 {
  346 
  347         KASSERT(td->td_kstack == 0,
  348             ("thread_alloc_stack called on a thread with kstack"));
  349         if (!vm_thread_new(td, pages))
  350                 return (0);
  351         cpu_thread_alloc(td);
  352         return (1);
  353 }
  354 
  355 /*
  356  * Deallocate a thread.
  357  */
  358 void
  359 thread_free(struct thread *td)
  360 {
  361 
  362         lock_profile_thread_exit(td);
  363         if (td->td_cpuset)
  364                 cpuset_rel(td->td_cpuset);
  365         td->td_cpuset = NULL;
  366         cpu_thread_free(td);
  367         if (td->td_kstack != 0)
  368                 vm_thread_dispose(td);
  369         uma_zfree(thread_zone, td);
  370 }
  371 
  372 /*
  373  * Discard the current thread and exit from its context.
  374  * Always called with scheduler locked.
  375  *
  376  * Because we can't free a thread while we're operating under its context,
  377  * push the current thread into our CPU's deadthread holder. This means
  378  * we needn't worry about someone else grabbing our context before we
  379  * do a cpu_throw().
  380  */
  381 void
  382 thread_exit(void)
  383 {
  384         uint64_t new_switchtime;
  385         struct thread *td;
  386         struct thread *td2;
  387         struct proc *p;
  388         int wakeup_swapper;
  389 
  390         td = curthread;
  391         p = td->td_proc;
  392 
  393         PROC_SLOCK_ASSERT(p, MA_OWNED);
  394         mtx_assert(&Giant, MA_NOTOWNED);
  395 
  396         PROC_LOCK_ASSERT(p, MA_OWNED);
  397         KASSERT(p != NULL, ("thread exiting without a process"));
  398         CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
  399             (long)p->p_pid, td->td_name);
  400         KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
  401 
  402 #ifdef AUDIT
  403         AUDIT_SYSCALL_EXIT(0, td);
  404 #endif
  405         umtx_thread_exit(td);
  406         /*
  407          * drop FPU & debug register state storage, or any other
  408          * architecture specific resources that
  409          * would not be on a new untouched process.
  410          */
  411         cpu_thread_exit(td);    /* XXXSMP */
  412 
  413         /* Do the same timestamp bookkeeping that mi_switch() would do. */
  414         new_switchtime = cpu_ticks();
  415         p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
  416         PCPU_SET(switchtime, new_switchtime);
  417         PCPU_SET(switchticks, ticks);
  418         PCPU_INC(cnt.v_swtch);
  419         /* Save our resource usage in our process. */
  420         td->td_ru.ru_nvcsw++;
  421         rucollect(&p->p_ru, &td->td_ru);
  422         /*
  423          * The last thread is left attached to the process
  424          * So that the whole bundle gets recycled. Skip
  425          * all this stuff if we never had threads.
  426          * EXIT clears all sign of other threads when
  427          * it goes to single threading, so the last thread always
  428          * takes the short path.
  429          */
  430         if (p->p_flag & P_HADTHREADS) {
  431                 if (p->p_numthreads > 1) {
  432                         thread_unlink(td);
  433                         td2 = FIRST_THREAD_IN_PROC(p);
  434                         sched_exit_thread(td2, td);
  435 
  436                         /*
  437                          * The test below is NOT true if we are the
  438                          * sole exiting thread. P_STOPPED_SINGLE is unset
  439                          * in exit1() after it is the only survivor.
  440                          */
  441                         if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  442                                 if (p->p_numthreads == p->p_suspcount) {
  443                                         thread_lock(p->p_singlethread);
  444                                         wakeup_swapper = thread_unsuspend_one(
  445                                                 p->p_singlethread);
  446                                         thread_unlock(p->p_singlethread);
  447                                         if (wakeup_swapper)
  448                                                 kick_proc0();
  449                                 }
  450                         }
  451 
  452                         atomic_add_int(&td->td_proc->p_exitthreads, 1);
  453                         PCPU_SET(deadthread, td);
  454                 } else {
  455                         /*
  456                          * The last thread is exiting.. but not through exit()
  457                          */
  458                         panic ("thread_exit: Last thread exiting on its own");
  459                 }
  460         } 
  461 #ifdef  HWPMC_HOOKS
  462         /*
  463          * If this thread is part of a process that is being tracked by hwpmc(4),
  464          * inform the module of the thread's impending exit.
  465          */
  466         if (PMC_PROC_IS_USING_PMCS(td->td_proc))
  467                 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
  468 #endif
  469         PROC_UNLOCK(p);
  470         ruxagg(p, td);
  471         thread_lock(td);
  472         PROC_SUNLOCK(p);
  473         td->td_state = TDS_INACTIVE;
  474 #ifdef WITNESS
  475         witness_thread_exit(td);
  476 #endif
  477         CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
  478         sched_throw(td);
  479         panic("I'm a teapot!");
  480         /* NOTREACHED */
  481 }
  482 
  483 /*
  484  * Do any thread specific cleanups that may be needed in wait()
  485  * called with Giant, proc and schedlock not held.
  486  */
  487 void
  488 thread_wait(struct proc *p)
  489 {
  490         struct thread *td;
  491 
  492         mtx_assert(&Giant, MA_NOTOWNED);
  493         KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
  494         td = FIRST_THREAD_IN_PROC(p);
  495         /* Lock the last thread so we spin until it exits cpu_throw(). */
  496         thread_lock(td);
  497         thread_unlock(td);
  498         /* Wait for any remaining threads to exit cpu_throw(). */
  499         while (p->p_exitthreads)
  500                 sched_relinquish(curthread);
  501         lock_profile_thread_exit(td);
  502         cpuset_rel(td->td_cpuset);
  503         td->td_cpuset = NULL;
  504         cpu_thread_clean(td);
  505         crfree(td->td_ucred);
  506         thread_reap();  /* check for zombie threads etc. */
  507 }
  508 
  509 /*
  510  * Link a thread to a process.
  511  * set up anything that needs to be initialized for it to
  512  * be used by the process.
  513  */
  514 void
  515 thread_link(struct thread *td, struct proc *p)
  516 {
  517 
  518         /*
  519          * XXX This can't be enabled because it's called for proc0 before
  520          * its lock has been created.
  521          * PROC_LOCK_ASSERT(p, MA_OWNED);
  522          */
  523         td->td_state    = TDS_INACTIVE;
  524         td->td_proc     = p;
  525         td->td_flags    = TDF_INMEM;
  526 
  527         LIST_INIT(&td->td_contested);
  528         LIST_INIT(&td->td_lprof[0]);
  529         LIST_INIT(&td->td_lprof[1]);
  530         sigqueue_init(&td->td_sigqueue, p);
  531         callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
  532         TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
  533         p->p_numthreads++;
  534 }
  535 
  536 /*
  537  * Convert a process with one thread to an unthreaded process.
  538  */
  539 void
  540 thread_unthread(struct thread *td)
  541 {
  542         struct proc *p = td->td_proc;
  543 
  544         KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
  545         p->p_flag &= ~P_HADTHREADS;
  546 }
  547 
  548 /*
  549  * Called from:
  550  *  thread_exit()
  551  */
  552 void
  553 thread_unlink(struct thread *td)
  554 {
  555         struct proc *p = td->td_proc;
  556 
  557         PROC_LOCK_ASSERT(p, MA_OWNED);
  558         TAILQ_REMOVE(&p->p_threads, td, td_plist);
  559         p->p_numthreads--;
  560         /* could clear a few other things here */
  561         /* Must  NOT clear links to proc! */
  562 }
  563 
  564 static int
  565 calc_remaining(struct proc *p, int mode)
  566 {
  567         int remaining;
  568 
  569         PROC_LOCK_ASSERT(p, MA_OWNED);
  570         PROC_SLOCK_ASSERT(p, MA_OWNED);
  571         if (mode == SINGLE_EXIT)
  572                 remaining = p->p_numthreads;
  573         else if (mode == SINGLE_BOUNDARY)
  574                 remaining = p->p_numthreads - p->p_boundary_count;
  575         else if (mode == SINGLE_NO_EXIT)
  576                 remaining = p->p_numthreads - p->p_suspcount;
  577         else
  578                 panic("calc_remaining: wrong mode %d", mode);
  579         return (remaining);
  580 }
  581 
  582 /*
  583  * Enforce single-threading.
  584  *
  585  * Returns 1 if the caller must abort (another thread is waiting to
  586  * exit the process or similar). Process is locked!
  587  * Returns 0 when you are successfully the only thread running.
  588  * A process has successfully single threaded in the suspend mode when
  589  * There are no threads in user mode. Threads in the kernel must be
  590  * allowed to continue until they get to the user boundary. They may even
  591  * copy out their return values and data before suspending. They may however be
  592  * accelerated in reaching the user boundary as we will wake up
  593  * any sleeping threads that are interruptable. (PCATCH).
  594  */
  595 int
  596 thread_single(int mode)
  597 {
  598         struct thread *td;
  599         struct thread *td2;
  600         struct proc *p;
  601         int remaining, wakeup_swapper;
  602 
  603         td = curthread;
  604         p = td->td_proc;
  605         mtx_assert(&Giant, MA_NOTOWNED);
  606         PROC_LOCK_ASSERT(p, MA_OWNED);
  607         KASSERT((td != NULL), ("curthread is NULL"));
  608 
  609         if ((p->p_flag & P_HADTHREADS) == 0)
  610                 return (0);
  611 
  612         /* Is someone already single threading? */
  613         if (p->p_singlethread != NULL && p->p_singlethread != td)
  614                 return (1);
  615 
  616         if (mode == SINGLE_EXIT) {
  617                 p->p_flag |= P_SINGLE_EXIT;
  618                 p->p_flag &= ~P_SINGLE_BOUNDARY;
  619         } else {
  620                 p->p_flag &= ~P_SINGLE_EXIT;
  621                 if (mode == SINGLE_BOUNDARY)
  622                         p->p_flag |= P_SINGLE_BOUNDARY;
  623                 else
  624                         p->p_flag &= ~P_SINGLE_BOUNDARY;
  625         }
  626         p->p_flag |= P_STOPPED_SINGLE;
  627         PROC_SLOCK(p);
  628         p->p_singlethread = td;
  629         remaining = calc_remaining(p, mode);
  630         while (remaining != 1) {
  631                 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
  632                         goto stopme;
  633                 wakeup_swapper = 0;
  634                 FOREACH_THREAD_IN_PROC(p, td2) {
  635                         if (td2 == td)
  636                                 continue;
  637                         thread_lock(td2);
  638                         td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
  639                         if (TD_IS_INHIBITED(td2)) {
  640                                 switch (mode) {
  641                                 case SINGLE_EXIT:
  642                                         if (TD_IS_SUSPENDED(td2))
  643                                                 wakeup_swapper |=
  644                                                     thread_unsuspend_one(td2);
  645                                         if (TD_ON_SLEEPQ(td2) &&
  646                                             (td2->td_flags & TDF_SINTR))
  647                                                 wakeup_swapper |=
  648                                                     sleepq_abort(td2, EINTR);
  649                                         break;
  650                                 case SINGLE_BOUNDARY:
  651                                         if (TD_IS_SUSPENDED(td2) &&
  652                                             !(td2->td_flags & TDF_BOUNDARY))
  653                                                 wakeup_swapper |=
  654                                                     thread_unsuspend_one(td2);
  655                                         if (TD_ON_SLEEPQ(td2) &&
  656                                             (td2->td_flags & TDF_SINTR))
  657                                                 wakeup_swapper |=
  658                                                     sleepq_abort(td2, ERESTART);
  659                                         break;
  660                                 case SINGLE_NO_EXIT:
  661                                         if (TD_IS_SUSPENDED(td2) &&
  662                                             !(td2->td_flags & TDF_BOUNDARY))
  663                                                 wakeup_swapper |=
  664                                                     thread_unsuspend_one(td2);
  665                                         if (TD_ON_SLEEPQ(td2) &&
  666                                             (td2->td_flags & TDF_SINTR))
  667                                                 wakeup_swapper |=
  668                                                     sleepq_abort(td2, ERESTART);
  669                                         break;
  670                                 default:
  671                                         break;
  672                                 }
  673                         }
  674 #ifdef SMP
  675                         else if (TD_IS_RUNNING(td2) && td != td2) {
  676                                 forward_signal(td2);
  677                         }
  678 #endif
  679                         thread_unlock(td2);
  680                 }
  681                 if (wakeup_swapper)
  682                         kick_proc0();
  683                 remaining = calc_remaining(p, mode);
  684 
  685                 /*
  686                  * Maybe we suspended some threads.. was it enough?
  687                  */
  688                 if (remaining == 1)
  689                         break;
  690 
  691 stopme:
  692                 /*
  693                  * Wake us up when everyone else has suspended.
  694                  * In the mean time we suspend as well.
  695                  */
  696                 thread_suspend_switch(td);
  697                 remaining = calc_remaining(p, mode);
  698         }
  699         if (mode == SINGLE_EXIT) {
  700                 /*
  701                  * We have gotten rid of all the other threads and we
  702                  * are about to either exit or exec. In either case,
  703                  * we try our utmost  to revert to being a non-threaded
  704                  * process.
  705                  */
  706                 p->p_singlethread = NULL;
  707                 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
  708                 thread_unthread(td);
  709         }
  710         PROC_SUNLOCK(p);
  711         return (0);
  712 }
  713 
  714 /*
  715  * Called in from locations that can safely check to see
  716  * whether we have to suspend or at least throttle for a
  717  * single-thread event (e.g. fork).
  718  *
  719  * Such locations include userret().
  720  * If the "return_instead" argument is non zero, the thread must be able to
  721  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
  722  *
  723  * The 'return_instead' argument tells the function if it may do a
  724  * thread_exit() or suspend, or whether the caller must abort and back
  725  * out instead.
  726  *
  727  * If the thread that set the single_threading request has set the
  728  * P_SINGLE_EXIT bit in the process flags then this call will never return
  729  * if 'return_instead' is false, but will exit.
  730  *
  731  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
  732  *---------------+--------------------+---------------------
  733  *       0       | returns 0          |   returns 0 or 1
  734  *               | when ST ends       |   immediatly
  735  *---------------+--------------------+---------------------
  736  *       1       | thread exits       |   returns 1
  737  *               |                    |  immediatly
  738  * 0 = thread_exit() or suspension ok,
  739  * other = return error instead of stopping the thread.
  740  *
  741  * While a full suspension is under effect, even a single threading
  742  * thread would be suspended if it made this call (but it shouldn't).
  743  * This call should only be made from places where
  744  * thread_exit() would be safe as that may be the outcome unless
  745  * return_instead is set.
  746  */
  747 int
  748 thread_suspend_check(int return_instead)
  749 {
  750         struct thread *td;
  751         struct proc *p;
  752         int wakeup_swapper;
  753 
  754         td = curthread;
  755         p = td->td_proc;
  756         mtx_assert(&Giant, MA_NOTOWNED);
  757         PROC_LOCK_ASSERT(p, MA_OWNED);
  758         while (P_SHOULDSTOP(p) ||
  759               ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) {
  760                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  761                         KASSERT(p->p_singlethread != NULL,
  762                             ("singlethread not set"));
  763                         /*
  764                          * The only suspension in action is a
  765                          * single-threading. Single threader need not stop.
  766                          * XXX Should be safe to access unlocked
  767                          * as it can only be set to be true by us.
  768                          */
  769                         if (p->p_singlethread == td)
  770                                 return (0);     /* Exempt from stopping. */
  771                 }
  772                 if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
  773                         return (EINTR);
  774 
  775                 /* Should we goto user boundary if we didn't come from there? */
  776                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
  777                     (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
  778                         return (ERESTART);
  779 
  780                 /*
  781                  * If the process is waiting for us to exit,
  782                  * this thread should just suicide.
  783                  * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
  784                  */
  785                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
  786                         PROC_UNLOCK(p);
  787                         tidhash_remove(td);
  788                         PROC_LOCK(p);
  789                         tdsigcleanup(td);
  790                         PROC_SLOCK(p);
  791                         thread_stopped(p);
  792                         thread_exit();
  793                 }
  794 
  795                 PROC_SLOCK(p);
  796                 thread_stopped(p);
  797                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  798                         if (p->p_numthreads == p->p_suspcount + 1) {
  799                                 thread_lock(p->p_singlethread);
  800                                 wakeup_swapper =
  801                                     thread_unsuspend_one(p->p_singlethread);
  802                                 thread_unlock(p->p_singlethread);
  803                                 if (wakeup_swapper)
  804                                         kick_proc0();
  805                         }
  806                 }
  807                 PROC_UNLOCK(p);
  808                 thread_lock(td);
  809                 /*
  810                  * When a thread suspends, it just
  811                  * gets taken off all queues.
  812                  */
  813                 thread_suspend_one(td);
  814                 if (return_instead == 0) {
  815                         p->p_boundary_count++;
  816                         td->td_flags |= TDF_BOUNDARY;
  817                 }
  818                 PROC_SUNLOCK(p);
  819                 mi_switch(SW_INVOL | SWT_SUSPEND, NULL);
  820                 if (return_instead == 0)
  821                         td->td_flags &= ~TDF_BOUNDARY;
  822                 thread_unlock(td);
  823                 PROC_LOCK(p);
  824                 if (return_instead == 0) {
  825                         PROC_SLOCK(p);
  826                         p->p_boundary_count--;
  827                         PROC_SUNLOCK(p);
  828                 }
  829         }
  830         return (0);
  831 }
  832 
  833 void
  834 thread_suspend_switch(struct thread *td)
  835 {
  836         struct proc *p;
  837 
  838         p = td->td_proc;
  839         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
  840         PROC_LOCK_ASSERT(p, MA_OWNED);
  841         PROC_SLOCK_ASSERT(p, MA_OWNED);
  842         /*
  843          * We implement thread_suspend_one in stages here to avoid
  844          * dropping the proc lock while the thread lock is owned.
  845          */
  846         thread_stopped(p);
  847         p->p_suspcount++;
  848         PROC_UNLOCK(p);
  849         thread_lock(td);
  850         td->td_flags &= ~TDF_NEEDSUSPCHK;
  851         TD_SET_SUSPENDED(td);
  852         sched_sleep(td, 0);
  853         PROC_SUNLOCK(p);
  854         DROP_GIANT();
  855         mi_switch(SW_VOL | SWT_SUSPEND, NULL);
  856         thread_unlock(td);
  857         PICKUP_GIANT();
  858         PROC_LOCK(p);
  859         PROC_SLOCK(p);
  860 }
  861 
  862 void
  863 thread_suspend_one(struct thread *td)
  864 {
  865         struct proc *p = td->td_proc;
  866 
  867         PROC_SLOCK_ASSERT(p, MA_OWNED);
  868         THREAD_LOCK_ASSERT(td, MA_OWNED);
  869         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
  870         p->p_suspcount++;
  871         td->td_flags &= ~TDF_NEEDSUSPCHK;
  872         TD_SET_SUSPENDED(td);
  873         sched_sleep(td, 0);
  874 }
  875 
  876 int
  877 thread_unsuspend_one(struct thread *td)
  878 {
  879         struct proc *p = td->td_proc;
  880 
  881         PROC_SLOCK_ASSERT(p, MA_OWNED);
  882         THREAD_LOCK_ASSERT(td, MA_OWNED);
  883         KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
  884         TD_CLR_SUSPENDED(td);
  885         p->p_suspcount--;
  886         return (setrunnable(td));
  887 }
  888 
  889 /*
  890  * Allow all threads blocked by single threading to continue running.
  891  */
  892 void
  893 thread_unsuspend(struct proc *p)
  894 {
  895         struct thread *td;
  896         int wakeup_swapper;
  897 
  898         PROC_LOCK_ASSERT(p, MA_OWNED);
  899         PROC_SLOCK_ASSERT(p, MA_OWNED);
  900         wakeup_swapper = 0;
  901         if (!P_SHOULDSTOP(p)) {
  902                 FOREACH_THREAD_IN_PROC(p, td) {
  903                         thread_lock(td);
  904                         if (TD_IS_SUSPENDED(td)) {
  905                                 wakeup_swapper |= thread_unsuspend_one(td);
  906                         }
  907                         thread_unlock(td);
  908                 }
  909         } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
  910             (p->p_numthreads == p->p_suspcount)) {
  911                 /*
  912                  * Stopping everything also did the job for the single
  913                  * threading request. Now we've downgraded to single-threaded,
  914                  * let it continue.
  915                  */
  916                 thread_lock(p->p_singlethread);
  917                 wakeup_swapper = thread_unsuspend_one(p->p_singlethread);
  918                 thread_unlock(p->p_singlethread);
  919         }
  920         if (wakeup_swapper)
  921                 kick_proc0();
  922 }
  923 
  924 /*
  925  * End the single threading mode..
  926  */
  927 void
  928 thread_single_end(void)
  929 {
  930         struct thread *td;
  931         struct proc *p;
  932         int wakeup_swapper;
  933 
  934         td = curthread;
  935         p = td->td_proc;
  936         PROC_LOCK_ASSERT(p, MA_OWNED);
  937         p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
  938         PROC_SLOCK(p);
  939         p->p_singlethread = NULL;
  940         wakeup_swapper = 0;
  941         /*
  942          * If there are other threads they may now run,
  943          * unless of course there is a blanket 'stop order'
  944          * on the process. The single threader must be allowed
  945          * to continue however as this is a bad place to stop.
  946          */
  947         if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
  948                 FOREACH_THREAD_IN_PROC(p, td) {
  949                         thread_lock(td);
  950                         if (TD_IS_SUSPENDED(td)) {
  951                                 wakeup_swapper |= thread_unsuspend_one(td);
  952                         }
  953                         thread_unlock(td);
  954                 }
  955         }
  956         PROC_SUNLOCK(p);
  957         if (wakeup_swapper)
  958                 kick_proc0();
  959 }
  960 
  961 struct thread *
  962 thread_find(struct proc *p, lwpid_t tid)
  963 {
  964         struct thread *td;
  965 
  966         PROC_LOCK_ASSERT(p, MA_OWNED);
  967         FOREACH_THREAD_IN_PROC(p, td) {
  968                 if (td->td_tid == tid)
  969                         break;
  970         }
  971         return (td);
  972 }
  973 
  974 /* Locate a thread by number; return with proc lock held. */
  975 struct thread *
  976 tdfind(lwpid_t tid, pid_t pid)
  977 {
  978 #define RUN_THRESH      16
  979         struct thread *td;
  980         int run = 0;
  981 
  982         rw_rlock(&tidhash_lock);
  983         LIST_FOREACH(td, TIDHASH(tid), td_hash) {
  984                 if (td->td_tid == tid) {
  985                         if (pid != -1 && td->td_proc->p_pid != pid) {
  986                                 td = NULL;
  987                                 break;
  988                         }
  989                         PROC_LOCK(td->td_proc);
  990                         if (td->td_proc->p_state == PRS_NEW) {
  991                                 PROC_UNLOCK(td->td_proc);
  992                                 td = NULL;
  993                                 break;
  994                         }
  995                         if (run > RUN_THRESH) {
  996                                 if (rw_try_upgrade(&tidhash_lock)) {
  997                                         LIST_REMOVE(td, td_hash);
  998                                         LIST_INSERT_HEAD(TIDHASH(td->td_tid),
  999                                                 td, td_hash);
 1000                                         rw_wunlock(&tidhash_lock);
 1001                                         return (td);
 1002                                 }
 1003                         }
 1004                         break;
 1005                 }
 1006                 run++;
 1007         }
 1008         rw_runlock(&tidhash_lock);
 1009         return (td);
 1010 }
 1011 
 1012 void
 1013 tidhash_add(struct thread *td)
 1014 {
 1015         rw_wlock(&tidhash_lock);
 1016         LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash);
 1017         rw_wunlock(&tidhash_lock);
 1018 }
 1019 
 1020 void
 1021 tidhash_remove(struct thread *td)
 1022 {
 1023         rw_wlock(&tidhash_lock);
 1024         LIST_REMOVE(td, td_hash);
 1025         rw_wunlock(&tidhash_lock);
 1026 }
Cache object: c0e40ea465dedf84bb127ef5f3ded819
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_thread.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_thread.c
