The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_thread.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
    3  *  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice(s), this list of conditions and the following disclaimer as
   10  *    the first lines of this file unmodified other than the possible
   11  *    addition of one or more copyright notices.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice(s), this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
   17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   19  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
   20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   23  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   26  * DAMAGE.
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD$");
   31 
   32 #include <sys/param.h>
   33 #include <sys/systm.h>
   34 #include <sys/kernel.h>
   35 #include <sys/lock.h>
   36 #include <sys/mutex.h>
   37 #include <sys/proc.h>
   38 #include <sys/resourcevar.h>
   39 #include <sys/smp.h>
   40 #include <sys/sysctl.h>
   41 #include <sys/sched.h>
   42 #include <sys/sleepqueue.h>
   43 #include <sys/turnstile.h>
   44 #include <sys/ktr.h>
   45 #include <sys/umtx.h>
   46 
   47 #include <security/audit/audit.h>
   48 
   49 #include <vm/vm.h>
   50 #include <vm/vm_extern.h>
   51 #include <vm/uma.h>
   52 
   53 /*
   54  * KSEGRP related storage.
   55  */
   56 static uma_zone_t ksegrp_zone;
   57 static uma_zone_t thread_zone;
   58 
   59 /* DEBUG ONLY */
   60 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
   61 static int thread_debug = 0;
   62 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW,
   63         &thread_debug, 0, "thread debug");
   64 
   65 int max_threads_per_proc = 1500;
   66 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
   67         &max_threads_per_proc, 0, "Limit on threads per proc");
   68 
   69 int max_groups_per_proc = 1500;
   70 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW,
   71         &max_groups_per_proc, 0, "Limit on thread groups per proc");
   72 
   73 int max_threads_hits;
   74 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
   75         &max_threads_hits, 0, "");
   76 
   77 int virtual_cpu;
   78 
   79 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
   80 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps);
   81 struct mtx kse_zombie_lock;
   82 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN);
   83 
   84 static int
   85 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS)
   86 {
   87         int error, new_val;
   88         int def_val;
   89 
   90         def_val = mp_ncpus;
   91         if (virtual_cpu == 0)
   92                 new_val = def_val;
   93         else
   94                 new_val = virtual_cpu;
   95         error = sysctl_handle_int(oidp, &new_val, 0, req);
   96         if (error != 0 || req->newptr == NULL)
   97                 return (error);
   98         if (new_val < 0)
   99                 return (EINVAL);
  100         virtual_cpu = new_val;
  101         return (0);
  102 }
  103 
  104 /* DEBUG ONLY */
  105 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW,
  106         0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I",
  107         "debug virtual cpus");
  108 
  109 struct mtx tid_lock;
  110 static struct unrhdr *tid_unrhdr;
  111 
  112 /*
  113  * Prepare a thread for use.
  114  */
  115 static int
  116 thread_ctor(void *mem, int size, void *arg, int flags)
  117 {
  118         struct thread   *td;
  119 
  120         td = (struct thread *)mem;
  121         td->td_state = TDS_INACTIVE;
  122         td->td_oncpu = NOCPU;
  123 
  124         td->td_tid = alloc_unr(tid_unrhdr);
  125 
  126         /*
  127          * Note that td_critnest begins life as 1 because the thread is not
  128          * running and is thereby implicitly waiting to be on the receiving
  129          * end of a context switch.  A context switch must occur inside a
  130          * critical section, and in fact, includes hand-off of the sched_lock.
  131          * After a context switch to a newly created thread, it will release
  132          * sched_lock for the first time, and its td_critnest will hit 0 for
  133          * the first time.  This happens on the far end of a context switch,
  134          * and when it context switches away from itself, it will in fact go
  135          * back into a critical section, and hand off the sched lock to the
  136          * next thread.
  137          */
  138         td->td_critnest = 1;
  139 
  140 #ifdef AUDIT
  141         audit_thread_alloc(td);
  142 #endif
  143         return (0);
  144 }
  145 
  146 /*
  147  * Reclaim a thread after use.
  148  */
  149 static void
  150 thread_dtor(void *mem, int size, void *arg)
  151 {
  152         struct thread *td;
  153 
  154         td = (struct thread *)mem;
  155 
  156 #ifdef INVARIANTS
  157         /* Verify that this thread is in a safe state to free. */
  158         switch (td->td_state) {
  159         case TDS_INHIBITED:
  160         case TDS_RUNNING:
  161         case TDS_CAN_RUN:
  162         case TDS_RUNQ:
  163                 /*
  164                  * We must never unlink a thread that is in one of
  165                  * these states, because it is currently active.
  166                  */
  167                 panic("bad state for thread unlinking");
  168                 /* NOTREACHED */
  169         case TDS_INACTIVE:
  170                 break;
  171         default:
  172                 panic("bad thread state");
  173                 /* NOTREACHED */
  174         }
  175 #endif
  176 #ifdef AUDIT
  177         audit_thread_free(td);
  178 #endif
  179         free_unr(tid_unrhdr, td->td_tid);
  180         sched_newthread(td);
  181 }
  182 
  183 /*
  184  * Initialize type-stable parts of a thread (when newly created).
  185  */
  186 static int
  187 thread_init(void *mem, int size, int flags)
  188 {
  189         struct thread *td;
  190 
  191         td = (struct thread *)mem;      
  192 
  193         vm_thread_new(td, 0);
  194         cpu_thread_setup(td);
  195         td->td_sleepqueue = sleepq_alloc();
  196         td->td_turnstile = turnstile_alloc();
  197         td->td_umtxq = umtxq_alloc();
  198         td->td_sched = (struct td_sched *)&td[1];
  199         sched_newthread(td);
  200         return (0);
  201 }
  202 
  203 /*
  204  * Tear down type-stable parts of a thread (just before being discarded).
  205  */
  206 static void
  207 thread_fini(void *mem, int size)
  208 {
  209         struct thread *td;
  210 
  211         td = (struct thread *)mem;
  212         turnstile_free(td->td_turnstile);
  213         sleepq_free(td->td_sleepqueue);
  214         umtxq_free(td->td_umtxq);
  215         vm_thread_dispose(td);
  216 }
  217 
  218 /*
  219  * Initialize type-stable parts of a ksegrp (when newly created).
  220  */
  221 static int
  222 ksegrp_ctor(void *mem, int size, void *arg, int flags)
  223 {
  224         struct ksegrp   *kg;
  225 
  226         kg = (struct ksegrp *)mem;
  227         bzero(mem, size);
  228         kg->kg_sched = (struct kg_sched *)&kg[1];
  229         return (0);
  230 }
  231 
  232 void
  233 ksegrp_link(struct ksegrp *kg, struct proc *p)
  234 {
  235 
  236         TAILQ_INIT(&kg->kg_threads);
  237         TAILQ_INIT(&kg->kg_runq);       /* links with td_runq */
  238         TAILQ_INIT(&kg->kg_upcalls);    /* all upcall structure in ksegrp */
  239         kg->kg_proc = p;
  240         /*
  241          * the following counters are in the -zero- section
  242          * and may not need clearing
  243          */
  244         kg->kg_numthreads = 0;
  245         kg->kg_numupcalls = 0;
  246         /* link it in now that it's consistent */
  247         p->p_numksegrps++;
  248         TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp);
  249 }
  250 
  251 /*
  252  * Called from:
  253  *   thread-exit()
  254  */
  255 void
  256 ksegrp_unlink(struct ksegrp *kg)
  257 {
  258         struct proc *p;
  259 
  260         mtx_assert(&sched_lock, MA_OWNED);
  261         KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads"));
  262         KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls"));
  263 
  264         p = kg->kg_proc;
  265         TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp);
  266         p->p_numksegrps--;
  267         /*
  268          * Aggregate stats from the KSE
  269          */
  270         if (p->p_procscopegrp == kg)
  271                 p->p_procscopegrp = NULL;
  272 }
  273 
  274 /*
  275  * For a newly created process,
  276  * link up all the structures and its initial threads etc.
  277  * called from:
  278  * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
  279  * proc_dtor() (should go away)
  280  * proc_init()
  281  */
  282 void
  283 proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td)
  284 {
  285 
  286         TAILQ_INIT(&p->p_ksegrps);           /* all ksegrps in proc */
  287         TAILQ_INIT(&p->p_threads);           /* all threads in proc */
  288         TAILQ_INIT(&p->p_suspended);         /* Threads suspended */
  289         p->p_numksegrps = 0;
  290         p->p_numthreads = 0;
  291 
  292         ksegrp_link(kg, p);
  293         thread_link(td, kg);
  294 }
  295 
  296 /*
  297  * Initialize global thread allocation resources.
  298  */
  299 void
  300 threadinit(void)
  301 {
  302 
  303         mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
  304         tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock);
  305 
  306         thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
  307             thread_ctor, thread_dtor, thread_init, thread_fini,
  308             THREAD_ALIGN - 1, 0);
  309         ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(),
  310             ksegrp_ctor, NULL, NULL, NULL,
  311             UMA_ALIGN_CACHE, 0);
  312         kseinit();      /* set up kse specific stuff  e.g. upcall zone*/
  313 }
  314 
  315 /*
  316  * Stash an embarasingly extra thread into the zombie thread queue.
  317  */
  318 void
  319 thread_stash(struct thread *td)
  320 {
  321         mtx_lock_spin(&kse_zombie_lock);
  322         TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq);
  323         mtx_unlock_spin(&kse_zombie_lock);
  324 }
  325 
  326 /*
  327  * Stash an embarasingly extra ksegrp into the zombie ksegrp queue.
  328  */
  329 void
  330 ksegrp_stash(struct ksegrp *kg)
  331 {
  332         mtx_lock_spin(&kse_zombie_lock);
  333         TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp);
  334         mtx_unlock_spin(&kse_zombie_lock);
  335 }
  336 
  337 /*
  338  * Reap zombie kse resource.
  339  */
  340 void
  341 thread_reap(void)
  342 {
  343         struct thread *td_first, *td_next;
  344         struct ksegrp *kg_first, * kg_next;
  345 
  346         /*
  347          * Don't even bother to lock if none at this instant,
  348          * we really don't care about the next instant..
  349          */
  350         if ((!TAILQ_EMPTY(&zombie_threads))
  351             || (!TAILQ_EMPTY(&zombie_ksegrps))) {
  352                 mtx_lock_spin(&kse_zombie_lock);
  353                 td_first = TAILQ_FIRST(&zombie_threads);
  354                 kg_first = TAILQ_FIRST(&zombie_ksegrps);
  355                 if (td_first)
  356                         TAILQ_INIT(&zombie_threads);
  357                 if (kg_first)
  358                         TAILQ_INIT(&zombie_ksegrps);
  359                 mtx_unlock_spin(&kse_zombie_lock);
  360                 while (td_first) {
  361                         td_next = TAILQ_NEXT(td_first, td_runq);
  362                         if (td_first->td_ucred)
  363                                 crfree(td_first->td_ucred);
  364                         thread_free(td_first);
  365                         td_first = td_next;
  366                 }
  367                 while (kg_first) {
  368                         kg_next = TAILQ_NEXT(kg_first, kg_ksegrp);
  369                         ksegrp_free(kg_first);
  370                         kg_first = kg_next;
  371                 }
  372                 /*
  373                  * there will always be a thread on the list if one of these
  374                  * is there.
  375                  */
  376                 kse_GC();
  377         }
  378 }
  379 
  380 /*
  381  * Allocate a ksegrp.
  382  */
  383 struct ksegrp *
  384 ksegrp_alloc(void)
  385 {
  386         return (uma_zalloc(ksegrp_zone, M_WAITOK));
  387 }
  388 
  389 /*
  390  * Allocate a thread.
  391  */
  392 struct thread *
  393 thread_alloc(void)
  394 {
  395         thread_reap(); /* check if any zombies to get */
  396         return (uma_zalloc(thread_zone, M_WAITOK));
  397 }
  398 
  399 /*
  400  * Deallocate a ksegrp.
  401  */
  402 void
  403 ksegrp_free(struct ksegrp *td)
  404 {
  405         uma_zfree(ksegrp_zone, td);
  406 }
  407 
  408 /*
  409  * Deallocate a thread.
  410  */
  411 void
  412 thread_free(struct thread *td)
  413 {
  414 
  415         cpu_thread_clean(td);
  416         uma_zfree(thread_zone, td);
  417 }
  418 
  419 /*
  420  * Discard the current thread and exit from its context.
  421  * Always called with scheduler locked.
  422  *
  423  * Because we can't free a thread while we're operating under its context,
  424  * push the current thread into our CPU's deadthread holder. This means
  425  * we needn't worry about someone else grabbing our context before we
  426  * do a cpu_throw().  This may not be needed now as we are under schedlock.
  427  * Maybe we can just do a thread_stash() as thr_exit1 does.
  428  */
  429 /*  XXX
  430  * libthr expects its thread exit to return for the last
  431  * thread, meaning that the program is back to non-threaded
  432  * mode I guess. Because we do this (cpu_throw) unconditionally
  433  * here, they have their own version of it. (thr_exit1()) 
  434  * that doesn't do it all if this was the last thread.
  435  * It is also called from thread_suspend_check().
  436  * Of course in the end, they end up coming here through exit1
  437  * anyhow..  After fixing 'thr' to play by the rules we should be able 
  438  * to merge these two functions together.
  439  *
  440  * called from:
  441  * exit1()
  442  * kse_exit()
  443  * thr_exit()
  444  * thread_user_enter()
  445  * thread_userret()
  446  * thread_suspend_check()
  447  */
  448 void
  449 thread_exit(void)
  450 {
  451         struct bintime new_switchtime;
  452         struct thread *td;
  453         struct proc *p;
  454         struct ksegrp   *kg;
  455 
  456         td = curthread;
  457         kg = td->td_ksegrp;
  458         p = td->td_proc;
  459 
  460         mtx_assert(&sched_lock, MA_OWNED);
  461         mtx_assert(&Giant, MA_NOTOWNED);
  462         PROC_LOCK_ASSERT(p, MA_OWNED);
  463         KASSERT(p != NULL, ("thread exiting without a process"));
  464         KASSERT(kg != NULL, ("thread exiting without a kse group"));
  465         CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
  466             (long)p->p_pid, p->p_comm);
  467 
  468 #ifdef AUDIT
  469         AUDIT_SYSCALL_EXIT(0, td);
  470 #endif
  471 
  472         if (td->td_standin != NULL) {
  473                 /*
  474                  * Note that we don't need to free the cred here as it
  475                  * is done in thread_reap().
  476                  */
  477                 thread_stash(td->td_standin);
  478                 td->td_standin = NULL;
  479         }
  480 
  481         /*
  482          * drop FPU & debug register state storage, or any other
  483          * architecture specific resources that
  484          * would not be on a new untouched process.
  485          */
  486         cpu_thread_exit(td);    /* XXXSMP */
  487 
  488         /*
  489          * The thread is exiting. scheduler can release its stuff
  490          * and collect stats etc.
  491          */
  492         sched_thread_exit(td);
  493 
  494         /* Do the same timestamp bookkeeping that mi_switch() would do. */
  495         binuptime(&new_switchtime);
  496         bintime_add(&p->p_rux.rux_runtime, &new_switchtime);
  497         bintime_sub(&p->p_rux.rux_runtime, PCPU_PTR(switchtime));
  498         PCPU_SET(switchtime, new_switchtime);
  499         PCPU_SET(switchticks, ticks);
  500         cnt.v_swtch++;
  501 
  502         /* Add our usage into the usage of all our children. */
  503         if (p->p_numthreads == 1)
  504                 ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux);
  505 
  506         /*
  507          * The last thread is left attached to the process
  508          * So that the whole bundle gets recycled. Skip
  509          * all this stuff if we never had threads.
  510          * EXIT clears all sign of other threads when
  511          * it goes to single threading, so the last thread always
  512          * takes the short path.
  513          */
  514         if (p->p_flag & P_HADTHREADS) {
  515                 if (p->p_numthreads > 1) {
  516                         thread_unlink(td);
  517 
  518                         /* XXX first arg not used in 4BSD or ULE */
  519                         sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
  520 
  521                         /*
  522                          * The test below is NOT true if we are the
  523                          * sole exiting thread. P_STOPPED_SNGL is unset
  524                          * in exit1() after it is the only survivor.
  525                          */
  526                         if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  527                                 if (p->p_numthreads == p->p_suspcount) {
  528                                         thread_unsuspend_one(p->p_singlethread);
  529                                 }
  530                         }
  531 
  532                         /*
  533                          * Because each upcall structure has an owner thread,
  534                          * owner thread exits only when process is in exiting
  535                          * state, so upcall to userland is no longer needed,
  536                          * deleting upcall structure is safe here.
  537                          * So when all threads in a group is exited, all upcalls
  538                          * in the group should be automatically freed.
  539                          *  XXXKSE This is a KSE thing and should be exported
  540                          * there somehow.
  541                          */
  542                         upcall_remove(td);
  543 
  544                         /*
  545                          * If the thread we unlinked above was the last one,
  546                          * then this ksegrp should go away too.
  547                          */
  548                         if (kg->kg_numthreads == 0) {
  549                                 /*
  550                                  * let the scheduler know about this in case
  551                                  * it needs to recover stats or resources.
  552                                  * Theoretically we could let
  553                                  * sched_exit_ksegrp()  do the equivalent of
  554                                  * setting the concurrency to 0
  555                                  * but don't do it yet to avoid changing
  556                                  * the existing scheduler code until we
  557                                  * are ready.
  558                                  * We supply a random other ksegrp
  559                                  * as the recipient of any built up
  560                                  * cpu usage etc. (If the scheduler wants it).
  561                                  * XXXKSE
  562                                  * This is probably not fair so think of
  563                                  * a better answer.
  564                                  */
  565                                 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td);
  566                                 sched_set_concurrency(kg, 0); /* XXX TEMP */
  567                                 ksegrp_unlink(kg);
  568                                 ksegrp_stash(kg);
  569                         }
  570                         PROC_UNLOCK(p);
  571                         td->td_ksegrp   = NULL;
  572                         PCPU_SET(deadthread, td);
  573                 } else {
  574                         /*
  575                          * The last thread is exiting.. but not through exit()
  576                          * what should we do?
  577                          * Theoretically this can't happen
  578                          * exit1() - clears threading flags before coming here
  579                          * kse_exit() - treats last thread specially
  580                          * thr_exit() - treats last thread specially
  581                          * thread_user_enter() - only if more exist
  582                          * thread_userret() - only if more exist
  583                          * thread_suspend_check() - only if more exist
  584                          */
  585                         panic ("thread_exit: Last thread exiting on its own");
  586                 }
  587         } else {
  588                 /*
  589                  * non threaded process comes here.
  590                  * This includes an EX threaded process that is coming
  591                  * here via exit1(). (exit1 dethreads the proc first).
  592                  */
  593                 PROC_UNLOCK(p);
  594         }
  595         td->td_state = TDS_INACTIVE;
  596         CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
  597         cpu_throw(td, choosethread());
  598         panic("I'm a teapot!");
  599         /* NOTREACHED */
  600 }
  601 
  602 /*
  603  * Do any thread specific cleanups that may be needed in wait()
  604  * called with Giant, proc and schedlock not held.
  605  */
  606 void
  607 thread_wait(struct proc *p)
  608 {
  609         struct thread *td;
  610 
  611         mtx_assert(&Giant, MA_NOTOWNED);
  612         KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
  613         KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()"));
  614         FOREACH_THREAD_IN_PROC(p, td) {
  615                 if (td->td_standin != NULL) {
  616                         if (td->td_standin->td_ucred != NULL) {
  617                                 crfree(td->td_standin->td_ucred);
  618                                 td->td_standin->td_ucred = NULL;
  619                         }
  620                         thread_free(td->td_standin);
  621                         td->td_standin = NULL;
  622                 }
  623                 cpu_thread_clean(td);
  624                 crfree(td->td_ucred);
  625         }
  626         thread_reap();  /* check for zombie threads etc. */
  627 }
  628 
  629 /*
  630  * Link a thread to a process.
  631  * set up anything that needs to be initialized for it to
  632  * be used by the process.
  633  *
  634  * Note that we do not link to the proc's ucred here.
  635  * The thread is linked as if running but no KSE assigned.
  636  * Called from:
  637  *  proc_linkup()
  638  *  thread_schedule_upcall()
  639  *  thr_create()
  640  */
  641 void
  642 thread_link(struct thread *td, struct ksegrp *kg)
  643 {
  644         struct proc *p;
  645 
  646         p = kg->kg_proc;
  647         td->td_state    = TDS_INACTIVE;
  648         td->td_proc     = p;
  649         td->td_ksegrp   = kg;
  650         td->td_flags    = 0;
  651         td->td_kflags   = 0;
  652 
  653         LIST_INIT(&td->td_contested);
  654         callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
  655         TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist);
  656         TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist);
  657         p->p_numthreads++;
  658         kg->kg_numthreads++;
  659 }
  660 
  661 /*
  662  * Convert a process with one thread to an unthreaded process.
  663  * Called from:
  664  *  thread_single(exit)  (called from execve and exit)
  665  *  kse_exit()          XXX may need cleaning up wrt KSE stuff
  666  */
  667 void
  668 thread_unthread(struct thread *td)
  669 {
  670         struct proc *p = td->td_proc;
  671 
  672         KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
  673         upcall_remove(td);
  674         p->p_flag &= ~(P_SA|P_HADTHREADS);
  675         td->td_mailbox = NULL;
  676         td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND);
  677         if (td->td_standin != NULL) {
  678                 thread_stash(td->td_standin);
  679                 td->td_standin = NULL;
  680         }
  681         sched_set_concurrency(td->td_ksegrp, 1);
  682 }
  683 
  684 /*
  685  * Called from:
  686  *  thread_exit()
  687  */
  688 void
  689 thread_unlink(struct thread *td)
  690 {
  691         struct proc *p = td->td_proc;
  692         struct ksegrp *kg = td->td_ksegrp;
  693 
  694         mtx_assert(&sched_lock, MA_OWNED);
  695         TAILQ_REMOVE(&p->p_threads, td, td_plist);
  696         p->p_numthreads--;
  697         TAILQ_REMOVE(&kg->kg_threads, td, td_kglist);
  698         kg->kg_numthreads--;
  699         /* could clear a few other things here */
  700         /* Must  NOT clear links to proc and ksegrp! */
  701 }
  702 
  703 /*
  704  * Enforce single-threading.
  705  *
  706  * Returns 1 if the caller must abort (another thread is waiting to
  707  * exit the process or similar). Process is locked!
  708  * Returns 0 when you are successfully the only thread running.
  709  * A process has successfully single threaded in the suspend mode when
  710  * There are no threads in user mode. Threads in the kernel must be
  711  * allowed to continue until they get to the user boundary. They may even
  712  * copy out their return values and data before suspending. They may however be
  713  * accellerated in reaching the user boundary as we will wake up
  714  * any sleeping threads that are interruptable. (PCATCH).
  715  */
  716 int
  717 thread_single(int mode)
  718 {
  719         struct thread *td;
  720         struct thread *td2;
  721         struct proc *p;
  722         int remaining;
  723 
  724         td = curthread;
  725         p = td->td_proc;
  726         mtx_assert(&Giant, MA_NOTOWNED);
  727         PROC_LOCK_ASSERT(p, MA_OWNED);
  728         KASSERT((td != NULL), ("curthread is NULL"));
  729 
  730         if ((p->p_flag & P_HADTHREADS) == 0)
  731                 return (0);
  732 
  733         /* Is someone already single threading? */
  734         if (p->p_singlethread != NULL && p->p_singlethread != td)
  735                 return (1);
  736 
  737         if (mode == SINGLE_EXIT) {
  738                 p->p_flag |= P_SINGLE_EXIT;
  739                 p->p_flag &= ~P_SINGLE_BOUNDARY;
  740         } else {
  741                 p->p_flag &= ~P_SINGLE_EXIT;
  742                 if (mode == SINGLE_BOUNDARY)
  743                         p->p_flag |= P_SINGLE_BOUNDARY;
  744                 else
  745                         p->p_flag &= ~P_SINGLE_BOUNDARY;
  746         }
  747         p->p_flag |= P_STOPPED_SINGLE;
  748         mtx_lock_spin(&sched_lock);
  749         p->p_singlethread = td;
  750         if (mode == SINGLE_EXIT)
  751                 remaining = p->p_numthreads;
  752         else if (mode == SINGLE_BOUNDARY)
  753                 remaining = p->p_numthreads - p->p_boundary_count;
  754         else
  755                 remaining = p->p_numthreads - p->p_suspcount;
  756         while (remaining != 1) {
  757                 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
  758                         goto stopme;
  759                 FOREACH_THREAD_IN_PROC(p, td2) {
  760                         if (td2 == td)
  761                                 continue;
  762                         td2->td_flags |= TDF_ASTPENDING;
  763                         if (TD_IS_INHIBITED(td2)) {
  764                                 switch (mode) {
  765                                 case SINGLE_EXIT:
  766                                         if (td->td_flags & TDF_DBSUSPEND)
  767                                                 td->td_flags &= ~TDF_DBSUSPEND;
  768                                         if (TD_IS_SUSPENDED(td2))
  769                                                 thread_unsuspend_one(td2);
  770                                         if (TD_ON_SLEEPQ(td2) &&
  771                                             (td2->td_flags & TDF_SINTR))
  772                                                 sleepq_abort(td2, EINTR);
  773                                         break;
  774                                 case SINGLE_BOUNDARY:
  775                                         if (TD_IS_SUSPENDED(td2) &&
  776                                             !(td2->td_flags & TDF_BOUNDARY))
  777                                                 thread_unsuspend_one(td2);
  778                                         if (TD_ON_SLEEPQ(td2) &&
  779                                             (td2->td_flags & TDF_SINTR))
  780                                                 sleepq_abort(td2, ERESTART);
  781                                         break;
  782                                 default:        
  783                                         if (TD_IS_SUSPENDED(td2))
  784                                                 continue;
  785                                         /*
  786                                          * maybe other inhibitted states too?
  787                                          */
  788                                         if ((td2->td_flags & TDF_SINTR) &&
  789                                             (td2->td_inhibitors &
  790                                             (TDI_SLEEPING | TDI_SWAPPED)))
  791                                                 thread_suspend_one(td2);
  792                                         break;
  793                                 }
  794                         }
  795 #ifdef SMP
  796                         else if (TD_IS_RUNNING(td2) && td != td2) {
  797                                 forward_signal(td2);
  798                         }
  799 #endif
  800                 }
  801                 if (mode == SINGLE_EXIT)
  802                         remaining = p->p_numthreads;
  803                 else if (mode == SINGLE_BOUNDARY)
  804                         remaining = p->p_numthreads - p->p_boundary_count;
  805                 else
  806                         remaining = p->p_numthreads - p->p_suspcount;
  807 
  808                 /*
  809                  * Maybe we suspended some threads.. was it enough?
  810                  */
  811                 if (remaining == 1)
  812                         break;
  813 
  814 stopme:
  815                 /*
  816                  * Wake us up when everyone else has suspended.
  817                  * In the mean time we suspend as well.
  818                  */
  819                 thread_stopped(p);
  820                 thread_suspend_one(td);
  821                 PROC_UNLOCK(p);
  822                 mi_switch(SW_VOL, NULL);
  823                 mtx_unlock_spin(&sched_lock);
  824                 PROC_LOCK(p);
  825                 mtx_lock_spin(&sched_lock);
  826                 if (mode == SINGLE_EXIT)
  827                         remaining = p->p_numthreads;
  828                 else if (mode == SINGLE_BOUNDARY)
  829                         remaining = p->p_numthreads - p->p_boundary_count;
  830                 else
  831                         remaining = p->p_numthreads - p->p_suspcount;
  832         }
  833         if (mode == SINGLE_EXIT) {
  834                 /*
  835                  * We have gotten rid of all the other threads and we
  836                  * are about to either exit or exec. In either case,
  837                  * we try our utmost  to revert to being a non-threaded
  838                  * process.
  839                  */
  840                 p->p_singlethread = NULL;
  841                 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
  842                 thread_unthread(td);
  843         }
  844         mtx_unlock_spin(&sched_lock);
  845         return (0);
  846 }
  847 
  848 /*
  849  * Called in from locations that can safely check to see
  850  * whether we have to suspend or at least throttle for a
  851  * single-thread event (e.g. fork).
  852  *
  853  * Such locations include userret().
  854  * If the "return_instead" argument is non zero, the thread must be able to
  855  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
  856  *
  857  * The 'return_instead' argument tells the function if it may do a
  858  * thread_exit() or suspend, or whether the caller must abort and back
  859  * out instead.
  860  *
  861  * If the thread that set the single_threading request has set the
  862  * P_SINGLE_EXIT bit in the process flags then this call will never return
  863  * if 'return_instead' is false, but will exit.
  864  *
  865  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
  866  *---------------+--------------------+---------------------
  867  *       0       | returns 0          |   returns 0 or 1
  868  *               | when ST ends       |   immediatly
  869  *---------------+--------------------+---------------------
  870  *       1       | thread exits       |   returns 1
  871  *               |                    |  immediatly
  872  * 0 = thread_exit() or suspension ok,
  873  * other = return error instead of stopping the thread.
  874  *
  875  * While a full suspension is under effect, even a single threading
  876  * thread would be suspended if it made this call (but it shouldn't).
  877  * This call should only be made from places where
  878  * thread_exit() would be safe as that may be the outcome unless
  879  * return_instead is set.
  880  */
  881 int
  882 thread_suspend_check(int return_instead)
  883 {
  884         struct thread *td;
  885         struct proc *p;
  886 
  887         td = curthread;
  888         p = td->td_proc;
  889         mtx_assert(&Giant, MA_NOTOWNED);
  890         PROC_LOCK_ASSERT(p, MA_OWNED);
  891         while (P_SHOULDSTOP(p) ||
  892               ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) {
  893                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  894                         KASSERT(p->p_singlethread != NULL,
  895                             ("singlethread not set"));
  896                         /*
  897                          * The only suspension in action is a
  898                          * single-threading. Single threader need not stop.
  899                          * XXX Should be safe to access unlocked
  900                          * as it can only be set to be true by us.
  901                          */
  902                         if (p->p_singlethread == td)
  903                                 return (0);     /* Exempt from stopping. */
  904                 }
  905                 if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
  906                         return (EINTR);
  907 
  908                 /* Should we goto user boundary if we didn't come from there? */
  909                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
  910                     (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
  911                         return (ERESTART);
  912 
  913                 mtx_lock_spin(&sched_lock);
  914                 thread_stopped(p);
  915                 /*
  916                  * If the process is waiting for us to exit,
  917                  * this thread should just suicide.
  918                  * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
  919                  */
  920                 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
  921                         thread_exit();
  922 
  923                 /*
  924                  * When a thread suspends, it just
  925                  * moves to the processes's suspend queue
  926                  * and stays there.
  927                  */
  928                 thread_suspend_one(td);
  929                 if (return_instead == 0) {
  930                         p->p_boundary_count++;
  931                         td->td_flags |= TDF_BOUNDARY;
  932                 }
  933                 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
  934                         if (p->p_numthreads == p->p_suspcount) 
  935                                 thread_unsuspend_one(p->p_singlethread);
  936                 }
  937                 PROC_UNLOCK(p);
  938                 mi_switch(SW_INVOL, NULL);
  939                 if (return_instead == 0) {
  940                         p->p_boundary_count--;
  941                         td->td_flags &= ~TDF_BOUNDARY;
  942                 }
  943                 mtx_unlock_spin(&sched_lock);
  944                 PROC_LOCK(p);
  945         }
  946         return (0);
  947 }
  948 
  949 void
  950 thread_suspend_one(struct thread *td)
  951 {
  952         struct proc *p = td->td_proc;
  953 
  954         mtx_assert(&sched_lock, MA_OWNED);
  955         PROC_LOCK_ASSERT(p, MA_OWNED);
  956         KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
  957         p->p_suspcount++;
  958         TD_SET_SUSPENDED(td);
  959         TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq);
  960 }
  961 
  962 void
  963 thread_unsuspend_one(struct thread *td)
  964 {
  965         struct proc *p = td->td_proc;
  966 
  967         mtx_assert(&sched_lock, MA_OWNED);
  968         PROC_LOCK_ASSERT(p, MA_OWNED);
  969         TAILQ_REMOVE(&p->p_suspended, td, td_runq);
  970         TD_CLR_SUSPENDED(td);
  971         p->p_suspcount--;
  972         setrunnable(td);
  973 }
  974 
  975 /*
  976  * Allow all threads blocked by single threading to continue running.
  977  */
  978 void
  979 thread_unsuspend(struct proc *p)
  980 {
  981         struct thread *td;
  982 
  983         mtx_assert(&sched_lock, MA_OWNED);
  984         PROC_LOCK_ASSERT(p, MA_OWNED);
  985         if (!P_SHOULDSTOP(p)) {
  986                 while ((td = TAILQ_FIRST(&p->p_suspended))) {
  987                         thread_unsuspend_one(td);
  988                 }
  989         } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
  990             (p->p_numthreads == p->p_suspcount)) {
  991                 /*
  992                  * Stopping everything also did the job for the single
  993                  * threading request. Now we've downgraded to single-threaded,
  994                  * let it continue.
  995                  */
  996                 thread_unsuspend_one(p->p_singlethread);
  997         }
  998 }
  999 
 1000 /*
 1001  * End the single threading mode..
 1002  */
 1003 void
 1004 thread_single_end(void)
 1005 {
 1006         struct thread *td;
 1007         struct proc *p;
 1008 
 1009         td = curthread;
 1010         p = td->td_proc;
 1011         PROC_LOCK_ASSERT(p, MA_OWNED);
 1012         p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
 1013         mtx_lock_spin(&sched_lock);
 1014         p->p_singlethread = NULL;
 1015         p->p_procscopegrp = NULL;
 1016         /*
 1017          * If there are other threads they mey now run,
 1018          * unless of course there is a blanket 'stop order'
 1019          * on the process. The single threader must be allowed
 1020          * to continue however as this is a bad place to stop.
 1021          */
 1022         if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
 1023                 while ((td = TAILQ_FIRST(&p->p_suspended))) {
 1024                         thread_unsuspend_one(td);
 1025                 }
 1026         }
 1027         mtx_unlock_spin(&sched_lock);
 1028 }
 1029 
 1030 /*
 1031  * Called before going into an interruptible sleep to see if we have been
 1032  * interrupted or requested to exit.
 1033  */
 1034 int
 1035 thread_sleep_check(struct thread *td)
 1036 {
 1037         struct proc *p;
 1038 
 1039         p = td->td_proc;
 1040         mtx_assert(&sched_lock, MA_OWNED);
 1041         if (p->p_flag & P_HADTHREADS) {
 1042                 if (p->p_singlethread != td) {
 1043                         if (p->p_flag & P_SINGLE_EXIT)
 1044                                 return (EINTR);
 1045                         if (p->p_flag & P_SINGLE_BOUNDARY)
 1046                                 return (ERESTART);
 1047                 }
 1048                 if (td->td_flags & TDF_INTERRUPT)
 1049                         return (td->td_intrval);
 1050         }
 1051         return (0);
 1052 }

Cache object: 158b1e5264c43c6c489ca8c82fae3886


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.