The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/common/os/lwp.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or http://www.opensolaris.org/os/licensing.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 
   22 /*
   23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
   24  * Use is subject to license terms.
   25  */
   26 
   27 #include <sys/param.h>
   28 #include <sys/types.h>
   29 #include <sys/sysmacros.h>
   30 #include <sys/systm.h>
   31 #include <sys/thread.h>
   32 #include <sys/proc.h>
   33 #include <sys/task.h>
   34 #include <sys/project.h>
   35 #include <sys/signal.h>
   36 #include <sys/errno.h>
   37 #include <sys/vmparam.h>
   38 #include <sys/stack.h>
   39 #include <sys/procfs.h>
   40 #include <sys/prsystm.h>
   41 #include <sys/cpuvar.h>
   42 #include <sys/kmem.h>
   43 #include <sys/vtrace.h>
   44 #include <sys/door.h>
   45 #include <vm/seg_kp.h>
   46 #include <sys/debug.h>
   47 #include <sys/tnf.h>
   48 #include <sys/schedctl.h>
   49 #include <sys/poll.h>
   50 #include <sys/copyops.h>
   51 #include <sys/lwp_upimutex_impl.h>
   52 #include <sys/cpupart.h>
   53 #include <sys/lgrp.h>
   54 #include <sys/rctl.h>
   55 #include <sys/contract_impl.h>
   56 #include <sys/cpc_impl.h>
   57 #include <sys/sdt.h>
   58 #include <sys/cmn_err.h>
   59 #include <sys/brand.h>
   60 #include <sys/cyclic.h>
   61 #include <sys/pool.h>
   62 
   63 /* hash function for the lwpid hash table, p->p_tidhash[] */
   64 #define TIDHASH(tid, hash_sz)   ((tid) & ((hash_sz) - 1))
   65 
   66 void *segkp_lwp;                /* cookie for pool of segkp resources */
   67 extern void reapq_move_lq_to_tq(kthread_t *);
   68 extern void freectx_ctx(struct ctxop *);
   69 
   70 /*
   71  * Create a kernel thread associated with a particular system process.  Give
   72  * it an LWP so that microstate accounting will be available for it.
   73  */
   74 kthread_t *
   75 lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
   76 {
   77         klwp_t *lwp;
   78 
   79         VERIFY((p->p_flag & SSYS) != 0);
   80 
   81         lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);
   82 
   83         VERIFY(lwp != NULL);
   84 
   85         return (lwptot(lwp));
   86 }
   87 
   88 /*
   89  * Create a thread that appears to be stopped at sys_rtt.
   90  */
   91 klwp_t *
   92 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
   93     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
   94 {
   95         klwp_t *lwp = NULL;
   96         kthread_t *t;
   97         kthread_t *tx;
   98         cpupart_t *oldpart = NULL;
   99         size_t  stksize;
  100         caddr_t lwpdata = NULL;
  101         processorid_t   binding;
  102         int err = 0;
  103         kproject_t *oldkpj, *newkpj;
  104         void *bufp = NULL;
  105         klwp_t *curlwp;
  106         lwpent_t *lep;
  107         lwpdir_t *old_dir = NULL;
  108         uint_t old_dirsz = 0;
  109         tidhash_t *old_hash = NULL;
  110         uint_t old_hashsz = 0;
  111         ret_tidhash_t *ret_tidhash = NULL;
  112         int i;
  113         int rctlfail = 0;
  114         boolean_t branded = 0;
  115         struct ctxop *ctx = NULL;
  116 
  117         ASSERT(cid != sysdccid);        /* system threads must start in SYS */
  118 
  119         ASSERT(p != &p0);               /* No new LWPs in p0. */
  120 
  121         mutex_enter(&p->p_lock);
  122         mutex_enter(&p->p_zone->zone_nlwps_lock);
  123         /*
  124          * don't enforce rctl limits on system processes
  125          */
  126         if (!CLASS_KERNEL(cid)) {
  127                 if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
  128                         if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
  129                             1, 0) & RCT_DENY)
  130                                 rctlfail = 1;
  131                 if (p->p_task->tk_proj->kpj_nlwps >=
  132                     p->p_task->tk_proj->kpj_nlwps_ctl)
  133                         if (rctl_test(rc_project_nlwps,
  134                             p->p_task->tk_proj->kpj_rctls, p, 1, 0)
  135                             & RCT_DENY)
  136                                 rctlfail = 1;
  137                 if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
  138                         if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
  139                             1, 0) & RCT_DENY)
  140                                 rctlfail = 1;
  141         }
  142         if (rctlfail) {
  143                 mutex_exit(&p->p_zone->zone_nlwps_lock);
  144                 mutex_exit(&p->p_lock);
  145                 return (NULL);
  146         }
  147         p->p_task->tk_nlwps++;
  148         p->p_task->tk_proj->kpj_nlwps++;
  149         p->p_zone->zone_nlwps++;
  150         mutex_exit(&p->p_zone->zone_nlwps_lock);
  151         mutex_exit(&p->p_lock);
  152 
  153         curlwp = ttolwp(curthread);
  154         if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
  155                 stksize = lwp_default_stksize;
  156 
  157         if (CLASS_KERNEL(cid)) {
  158                 /*
  159                  * Since we are creating an LWP in an SSYS process, we do not
  160                  * inherit anything from the current thread's LWP.  We set
  161                  * stksize and lwpdata to 0 in order to let thread_create()
  162                  * allocate a regular kernel thread stack for this thread.
  163                  */
  164                 curlwp = NULL;
  165                 stksize = 0;
  166                 lwpdata = NULL;
  167 
  168         } else if (stksize == lwp_default_stksize) {
  169                 /*
  170                  * Try to reuse an <lwp,stack> from the LWP deathrow.
  171                  */
  172                 if (lwp_reapcnt > 0) {
  173                         mutex_enter(&reaplock);
  174                         if ((t = lwp_deathrow) != NULL) {
  175                                 ASSERT(t->t_swap);
  176                                 lwp_deathrow = t->t_forw;
  177                                 lwp_reapcnt--;
  178                                 lwpdata = t->t_swap;
  179                                 lwp = t->t_lwp;
  180                                 ctx = t->t_ctx;
  181                                 t->t_swap = NULL;
  182                                 t->t_lwp = NULL;
  183                                 t->t_ctx = NULL;
  184                                 reapq_move_lq_to_tq(t);
  185                         }
  186                         mutex_exit(&reaplock);
  187                         if (lwp != NULL) {
  188                                 lwp_stk_fini(lwp);
  189                         }
  190                         if (ctx != NULL) {
  191                                 freectx_ctx(ctx);
  192                         }
  193                 }
  194                 if (lwpdata == NULL &&
  195                     (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
  196                         mutex_enter(&p->p_lock);
  197                         mutex_enter(&p->p_zone->zone_nlwps_lock);
  198                         p->p_task->tk_nlwps--;
  199                         p->p_task->tk_proj->kpj_nlwps--;
  200                         p->p_zone->zone_nlwps--;
  201                         mutex_exit(&p->p_zone->zone_nlwps_lock);
  202                         mutex_exit(&p->p_lock);
  203                         return (NULL);
  204                 }
  205         } else {
  206                 stksize = roundup(stksize, PAGESIZE);
  207                 if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
  208                     (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
  209                         mutex_enter(&p->p_lock);
  210                         mutex_enter(&p->p_zone->zone_nlwps_lock);
  211                         p->p_task->tk_nlwps--;
  212                         p->p_task->tk_proj->kpj_nlwps--;
  213                         p->p_zone->zone_nlwps--;
  214                         mutex_exit(&p->p_zone->zone_nlwps_lock);
  215                         mutex_exit(&p->p_lock);
  216                         return (NULL);
  217                 }
  218         }
  219 
  220         /*
  221          * Create a thread, initializing the stack pointer
  222          */
  223         t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
  224 
  225         /*
  226          * If a non-NULL stack base is passed in, thread_create() assumes
  227          * that the stack might be statically allocated (as opposed to being
  228          * allocated from segkp), and so it does not set t_swap.  Since
  229          * the lwpdata was allocated from segkp, we must set t_swap to point
  230          * to it ourselves.
  231          *
  232          * This would be less confusing if t_swap had a better name; it really
  233          * indicates that the stack is allocated from segkp, regardless of
  234          * whether or not it is swappable.
  235          */
  236         if (lwpdata != NULL) {
  237                 ASSERT(!CLASS_KERNEL(cid));
  238                 ASSERT(t->t_swap == NULL);
  239                 t->t_swap = lwpdata;    /* Start of page-able data */
  240         }
  241 
  242         /*
  243          * If the stack and lwp can be reused, mark the thread as such.
  244          * When we get to reapq_add() from resume_from_zombie(), these
  245          * threads will go onto lwp_deathrow instead of thread_deathrow.
  246          */
  247         if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
  248                 t->t_flag |= T_LWPREUSE;
  249 
  250         if (lwp == NULL)
  251                 lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
  252         bzero(lwp, sizeof (*lwp));
  253         t->t_lwp = lwp;
  254 
  255         t->t_hold = *smask;
  256         lwp->lwp_thread = t;
  257         lwp->lwp_procp = p;
  258         lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
  259         if (curlwp != NULL && curlwp->lwp_childstksz != 0)
  260                 lwp->lwp_childstksz = curlwp->lwp_childstksz;
  261 
  262         t->t_stk = lwp_stk_init(lwp, t->t_stk);
  263         thread_load(t, proc, arg, len);
  264 
  265         /*
  266          * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
  267          */
  268         if (p->p_rprof_cyclic != CYCLIC_NONE)
  269                 t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
  270 
  271         if (cid != NOCLASS)
  272                 (void) CL_ALLOC(&bufp, cid, KM_SLEEP);
  273 
  274         /*
  275          * Allocate an lwp directory entry for the new lwp.
  276          */
  277         lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
  278 
  279         mutex_enter(&p->p_lock);
  280 grow:
  281         /*
  282          * Grow the lwp (thread) directory and lwpid hash table if necessary.
  283          * A note on the growth algorithm:
  284          *      The new lwp directory size is computed as:
  285          *              new = 2 * old + 2
  286          *      Starting with an initial size of 2 (see exec_common()),
  287          *      this yields numbers that are a power of two minus 2:
  288          *              2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
  289          *      The size of the lwpid hash table must be a power of two
  290          *      and must be commensurate in size with the lwp directory
  291          *      so that hash bucket chains remain short.  Therefore,
  292          *      the lwpid hash table size is computed as:
  293          *              hashsz = (dirsz + 2) / 2
  294          *      which leads to these hash table sizes corresponding to
  295          *      the above directory sizes:
  296          *              2, 4, 8, 16, 32, 64, 128, 256, 512, ...
  297          * A note on growing the hash table:
  298          *      For performance reasons, code in lwp_unpark() does not
  299          *      acquire curproc->p_lock when searching the hash table.
  300          *      Rather, it calls lwp_hash_lookup_and_lock() which
  301          *      acquires only the individual hash bucket lock, taking
  302          *      care to deal with reallocation of the hash table
  303          *      during the time it takes to acquire the lock.
  304          *
  305          *      This is sufficient to protect the integrity of the
  306          *      hash table, but it requires us to acquire all of the
  307          *      old hash bucket locks before growing the hash table
  308          *      and to release them afterwards.  It also requires us
  309          *      not to free the old hash table because some thread
  310          *      in lwp_hash_lookup_and_lock() might still be trying
  311          *      to acquire the old bucket lock.
  312          *
  313          *      So we adopt the tactic of keeping all of the retired
  314          *      hash tables on a linked list, so they can be safely
  315          *      freed when the process exits or execs.
  316          *
  317          *      Because the hash table grows in powers of two, the
  318          *      total size of all of the hash tables will be slightly
  319          *      less than twice the size of the largest hash table.
  320          */
  321         while (p->p_lwpfree == NULL) {
  322                 uint_t dirsz = p->p_lwpdir_sz;
  323                 lwpdir_t *new_dir;
  324                 uint_t new_dirsz;
  325                 lwpdir_t *ldp;
  326                 tidhash_t *new_hash;
  327                 uint_t new_hashsz;
  328 
  329                 mutex_exit(&p->p_lock);
  330 
  331                 /*
  332                  * Prepare to remember the old p_tidhash for later
  333                  * kmem_free()ing when the process exits or execs.
  334                  */
  335                 if (ret_tidhash == NULL)
  336                         ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
  337                             KM_SLEEP);
  338                 if (old_dir != NULL)
  339                         kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
  340                 if (old_hash != NULL)
  341                         kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
  342 
  343                 new_dirsz = 2 * dirsz + 2;
  344                 new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
  345                 for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
  346                         ldp->ld_next = ldp + 1;
  347                 new_hashsz = (new_dirsz + 2) / 2;
  348                 new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
  349                     KM_SLEEP);
  350 
  351                 mutex_enter(&p->p_lock);
  352                 if (p == curproc)
  353                         prbarrier(p);
  354 
  355                 if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
  356                         /*
  357                          * Someone else beat us to it or some lwp exited.
  358                          * Set up to free our memory and take a lap.
  359                          */
  360                         old_dir = new_dir;
  361                         old_dirsz = new_dirsz;
  362                         old_hash = new_hash;
  363                         old_hashsz = new_hashsz;
  364                 } else {
  365                         /*
  366                          * For the benefit of lwp_hash_lookup_and_lock(),
  367                          * called from lwp_unpark(), which searches the
  368                          * tid hash table without acquiring p->p_lock,
  369                          * we must acquire all of the tid hash table
  370                          * locks before replacing p->p_tidhash.
  371                          */
  372                         old_hash = p->p_tidhash;
  373                         old_hashsz = p->p_tidhash_sz;
  374                         for (i = 0; i < old_hashsz; i++) {
  375                                 mutex_enter(&old_hash[i].th_lock);
  376                                 mutex_enter(&new_hash[i].th_lock);
  377                         }
  378 
  379                         /*
  380                          * We simply hash in all of the old directory entries.
  381                          * This works because the old directory has no empty
  382                          * slots and the new hash table starts out empty.
  383                          * This reproduces the original directory ordering
  384                          * (required for /proc directory semantics).
  385                          */
  386                         old_dir = p->p_lwpdir;
  387                         old_dirsz = p->p_lwpdir_sz;
  388                         p->p_lwpdir = new_dir;
  389                         p->p_lwpfree = new_dir;
  390                         p->p_lwpdir_sz = new_dirsz;
  391                         for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
  392                                 lwp_hash_in(p, ldp->ld_entry,
  393                                     new_hash, new_hashsz, 0);
  394 
  395                         /*
  396                          * Remember the old hash table along with all
  397                          * of the previously-remembered hash tables.
  398                          * We will free them at process exit or exec.
  399                          */
  400                         ret_tidhash->rth_tidhash = old_hash;
  401                         ret_tidhash->rth_tidhash_sz = old_hashsz;
  402                         ret_tidhash->rth_next = p->p_ret_tidhash;
  403                         p->p_ret_tidhash = ret_tidhash;
  404 
  405                         /*
  406                          * Now establish the new tid hash table.
  407                          * As soon as we assign p->p_tidhash,
  408                          * code in lwp_unpark() can start using it.
  409                          */
  410                         membar_producer();
  411                         p->p_tidhash = new_hash;
  412 
  413                         /*
  414                          * It is necessary that p_tidhash reach global
  415                          * visibility before p_tidhash_sz.  Otherwise,
  416                          * code in lwp_hash_lookup_and_lock() could
  417                          * index into the old p_tidhash using the new
  418                          * p_tidhash_sz and thereby access invalid data.
  419                          */
  420                         membar_producer();
  421                         p->p_tidhash_sz = new_hashsz;
  422 
  423                         /*
  424                          * Release the locks; allow lwp_unpark() to carry on.
  425                          */
  426                         for (i = 0; i < old_hashsz; i++) {
  427                                 mutex_exit(&old_hash[i].th_lock);
  428                                 mutex_exit(&new_hash[i].th_lock);
  429                         }
  430 
  431                         /*
  432                          * Avoid freeing these objects below.
  433                          */
  434                         ret_tidhash = NULL;
  435                         old_hash = NULL;
  436                         old_hashsz = 0;
  437                 }
  438         }
  439 
  440         /*
  441          * Block the process against /proc while we manipulate p->p_tlist,
  442          * unless lwp_create() was called by /proc for the PCAGENT operation.
  443          * We want to do this early enough so that we don't drop p->p_lock
  444          * until the thread is put on the p->p_tlist.
  445          */
  446         if (p == curproc) {
  447                 prbarrier(p);
  448                 /*
  449                  * If the current lwp has been requested to stop, do so now.
  450                  * Otherwise we have a race condition between /proc attempting
  451                  * to stop the process and this thread creating a new lwp
  452                  * that was not seen when the /proc PCSTOP request was issued.
  453                  * We rely on stop() to call prbarrier(p) before returning.
  454                  */
  455                 while ((curthread->t_proc_flag & TP_PRSTOP) &&
  456                     !ttolwp(curthread)->lwp_nostop) {
  457                         /*
  458                          * We called pool_barrier_enter() before calling
  459                          * here to lwp_create(). We have to call
  460                          * pool_barrier_exit() before stopping.
  461                          */
  462                         pool_barrier_exit();
  463                         prbarrier(p);
  464                         stop(PR_REQUESTED, 0);
  465                         /*
  466                          * And we have to repeat the call to
  467                          * pool_barrier_enter after stopping.
  468                          */
  469                         pool_barrier_enter();
  470                         prbarrier(p);
  471                 }
  472 
  473                 /*
  474                  * If process is exiting, there could be a race between
  475                  * the agent lwp creation and the new lwp currently being
  476                  * created. So to prevent this race lwp creation is failed
  477                  * if the process is exiting.
  478                  */
  479                 if (p->p_flag & (SEXITLWPS|SKILLED)) {
  480                         err = 1;
  481                         goto error;
  482                 }
  483 
  484                 /*
  485                  * Since we might have dropped p->p_lock, the
  486                  * lwp directory free list might have changed.
  487                  */
  488                 if (p->p_lwpfree == NULL)
  489                         goto grow;
  490         }
  491 
  492         kpreempt_disable();     /* can't grab cpu_lock here */
  493 
  494         /*
  495          * Inherit processor and processor set bindings from curthread.
  496          *
  497          * For kernel LWPs, we do not inherit processor set bindings at
  498          * process creation time (i.e. when p != curproc).  After the
  499          * kernel process is created, any subsequent LWPs must be created
  500          * by threads in the kernel process, at which point we *will*
  501          * inherit processor set bindings.
  502          */
  503         if (CLASS_KERNEL(cid) && p != curproc) {
  504                 t->t_bind_cpu = binding = PBIND_NONE;
  505                 t->t_cpupart = oldpart = &cp_default;
  506                 t->t_bind_pset = PS_NONE;
  507                 t->t_bindflag = (uchar_t)default_binding_mode;
  508         } else {
  509                 binding = curthread->t_bind_cpu;
  510                 t->t_bind_cpu = binding;
  511                 oldpart = t->t_cpupart;
  512                 t->t_cpupart = curthread->t_cpupart;
  513                 t->t_bind_pset = curthread->t_bind_pset;
  514                 t->t_bindflag = curthread->t_bindflag |
  515                     (uchar_t)default_binding_mode;
  516         }
  517 
  518         /*
  519          * thread_create() initializes this thread's home lgroup to the root.
  520          * Choose a more suitable lgroup, since this thread is associated
  521          * with an lwp.
  522          */
  523         ASSERT(oldpart != NULL);
  524         if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
  525                 t->t_bound_cpu = cpu[binding];
  526                 if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
  527                         lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
  528         } else if (CLASS_KERNEL(cid)) {
  529                 /*
  530                  * Kernel threads are always in the root lgrp.
  531                  */
  532                 lgrp_move_thread(t,
  533                     &t->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
  534         } else {
  535                 lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
  536         }
  537 
  538         kpreempt_enable();
  539 
  540         /*
  541          * make sure lpl points to our own partition
  542          */
  543         ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
  544         ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
  545             t->t_cpupart->cp_nlgrploads);
  546 
  547         /*
  548          * It is safe to point the thread to the new project without holding it
  549          * since we're holding the target process' p_lock here and therefore
  550          * we're guaranteed that it will not move to another project.
  551          */
  552         newkpj = p->p_task->tk_proj;
  553         oldkpj = ttoproj(t);
  554         if (newkpj != oldkpj) {
  555                 t->t_proj = newkpj;
  556                 (void) project_hold(newkpj);
  557                 project_rele(oldkpj);
  558         }
  559 
  560         if (cid != NOCLASS) {
  561                 /*
  562                  * If the lwp is being created in the current process
  563                  * and matches the current thread's scheduling class,
  564                  * we should propagate the current thread's scheduling
  565                  * parameters by calling CL_FORK.  Otherwise just use
  566                  * the defaults by calling CL_ENTERCLASS.
  567                  */
  568                 if (p != curproc || curthread->t_cid != cid) {
  569                         err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
  570                         t->t_pri = pri; /* CL_ENTERCLASS may have changed it */
  571                         /*
  572                          * We don't call schedctl_set_cidpri(t) here
  573                          * because the schedctl data is not yet set
  574                          * up for the newly-created lwp.
  575                          */
  576                 } else {
  577                         t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
  578                         err = CL_FORK(curthread, t, bufp);
  579                         t->t_cid = cid;
  580                 }
  581                 if (err)
  582                         goto error;
  583                 else
  584                         bufp = NULL;
  585         }
  586 
  587         /*
  588          * If we were given an lwpid then use it, else allocate one.
  589          */
  590         if (lwpid != 0)
  591                 t->t_tid = lwpid;
  592         else {
  593                 /*
  594                  * lwp/thread id 0 is never valid; reserved for special checks.
  595                  * lwp/thread id 1 is reserved for the main thread.
  596                  * Start again at 2 when INT_MAX has been reached
  597                  * (id_t is a signed 32-bit integer).
  598                  */
  599                 id_t prev_id = p->p_lwpid;      /* last allocated tid */
  600 
  601                 do {                    /* avoid lwpid duplication */
  602                         if (p->p_lwpid == INT_MAX) {
  603                                 p->p_flag |= SLWPWRAP;
  604                                 p->p_lwpid = 1;
  605                         }
  606                         if ((t->t_tid = ++p->p_lwpid) == prev_id) {
  607                                 /*
  608                                  * All lwpids are allocated; fail the request.
  609                                  */
  610                                 err = 1;
  611                                 goto error;
  612                         }
  613                         /*
  614                          * We only need to worry about colliding with an id
  615                          * that's already in use if this process has
  616                          * cycled through all available lwp ids.
  617                          */
  618                         if ((p->p_flag & SLWPWRAP) == 0)
  619                                 break;
  620                 } while (lwp_hash_lookup(p, t->t_tid) != NULL);
  621         }
  622 
  623         /*
  624          * If this is a branded process, let the brand do any necessary lwp
  625          * initialization.
  626          */
  627         if (PROC_IS_BRANDED(p)) {
  628                 if (BROP(p)->b_initlwp(lwp)) {
  629                         err = 1;
  630                         goto error;
  631                 }
  632                 branded = 1;
  633         }
  634 
  635         if (t->t_tid == 1) {
  636                 kpreempt_disable();
  637                 ASSERT(t->t_lpl != NULL);
  638                 p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
  639                 kpreempt_enable();
  640                 if (p->p_tr_lgrpid != LGRP_NONE &&
  641                     p->p_tr_lgrpid != p->p_t1_lgrpid) {
  642                         lgrp_update_trthr_migrations(1);
  643                 }
  644         }
  645 
  646         p->p_lwpcnt++;
  647         t->t_waitfor = -1;
  648 
  649         /*
  650          * Turn microstate accounting on for thread if on for process.
  651          */
  652         if (p->p_flag & SMSACCT)
  653                 t->t_proc_flag |= TP_MSACCT;
  654 
  655         /*
  656          * If the process has watchpoints, mark the new thread as such.
  657          */
  658         if (pr_watch_active(p))
  659                 watch_enable(t);
  660 
  661         /*
  662          * The lwp is being created in the stopped state.
  663          * We set all the necessary flags to indicate that fact here.
  664          * We omit the TS_CREATE flag from t_schedflag so that the lwp
  665          * cannot be set running until the caller is finished with it,
  666          * even if lwp_continue() is called on it after we drop p->p_lock.
  667          * When the caller is finished with the newly-created lwp,
  668          * the caller must call lwp_create_done() to allow the lwp
  669          * to be set running.  If the TP_HOLDLWP is left set, the
  670          * lwp will suspend itself after reaching system call exit.
  671          */
  672         init_mstate(t, LMS_STOPPED);
  673         t->t_proc_flag |= TP_HOLDLWP;
  674         t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
  675         t->t_whystop = PR_SUSPENDED;
  676         t->t_whatstop = SUSPEND_NORMAL;
  677         t->t_sig_check = 1;     /* ensure that TP_HOLDLWP is honored */
  678 
  679         /*
  680          * Set system call processing flags in case tracing or profiling
  681          * is set.  The first system call will evaluate these and turn
  682          * them off if they aren't needed.
  683          */
  684         t->t_pre_sys = 1;
  685         t->t_post_sys = 1;
  686 
  687         /*
  688          * Insert the new thread into the list of all threads.
  689          */
  690         if ((tx = p->p_tlist) == NULL) {
  691                 t->t_back = t;
  692                 t->t_forw = t;
  693                 p->p_tlist = t;
  694         } else {
  695                 t->t_forw = tx;
  696                 t->t_back = tx->t_back;
  697                 tx->t_back->t_forw = t;
  698                 tx->t_back = t;
  699         }
  700 
  701         /*
  702          * Insert the new lwp into an lwp directory slot position
  703          * and into the lwpid hash table.
  704          */
  705         lep->le_thread = t;
  706         lep->le_lwpid = t->t_tid;
  707         lep->le_start = t->t_start;
  708         lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
  709 
  710         if (state == TS_RUN) {
  711                 /*
  712                  * We set the new lwp running immediately.
  713                  */
  714                 t->t_proc_flag &= ~TP_HOLDLWP;
  715                 lwp_create_done(t);
  716         }
  717 
  718 error:
  719         if (err) {
  720                 if (CLASS_KERNEL(cid)) {
  721                         /*
  722                          * This should only happen if a system process runs
  723                          * out of lwpids, which shouldn't occur.
  724                          */
  725                         panic("Failed to create a system LWP");
  726                 }
  727                 /*
  728                  * We have failed to create an lwp, so decrement the number
  729                  * of lwps in the task and let the lgroup load averages know
  730                  * that this thread isn't going to show up.
  731                  */
  732                 kpreempt_disable();
  733                 lgrp_move_thread(t, NULL, 1);
  734                 kpreempt_enable();
  735 
  736                 ASSERT(MUTEX_HELD(&p->p_lock));
  737                 mutex_enter(&p->p_zone->zone_nlwps_lock);
  738                 p->p_task->tk_nlwps--;
  739                 p->p_task->tk_proj->kpj_nlwps--;
  740                 p->p_zone->zone_nlwps--;
  741                 mutex_exit(&p->p_zone->zone_nlwps_lock);
  742                 if (cid != NOCLASS && bufp != NULL)
  743                         CL_FREE(cid, bufp);
  744 
  745                 if (branded)
  746                         BROP(p)->b_freelwp(lwp);
  747 
  748                 mutex_exit(&p->p_lock);
  749                 t->t_state = TS_FREE;
  750                 thread_rele(t);
  751 
  752                 /*
  753                  * We need to remove t from the list of all threads
  754                  * because thread_exit()/lwp_exit() isn't called on t.
  755                  */
  756                 mutex_enter(&pidlock);
  757                 ASSERT(t != t->t_next);         /* t0 never exits */
  758                 t->t_next->t_prev = t->t_prev;
  759                 t->t_prev->t_next = t->t_next;
  760                 mutex_exit(&pidlock);
  761 
  762                 thread_free(t);
  763                 kmem_free(lep, sizeof (*lep));
  764                 lwp = NULL;
  765         } else {
  766                 mutex_exit(&p->p_lock);
  767         }
  768 
  769         if (old_dir != NULL)
  770                 kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
  771         if (old_hash != NULL)
  772                 kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
  773         if (ret_tidhash != NULL)
  774                 kmem_free(ret_tidhash, sizeof (ret_tidhash_t));
  775 
  776         DTRACE_PROC1(lwp__create, kthread_t *, t);
  777         return (lwp);
  778 }
  779 
  780 /*
  781  * lwp_create_done() is called by the caller of lwp_create() to set the
  782  * newly-created lwp running after the caller has finished manipulating it.
  783  */
  784 void
  785 lwp_create_done(kthread_t *t)
  786 {
  787         proc_t *p = ttoproc(t);
  788 
  789         ASSERT(MUTEX_HELD(&p->p_lock));
  790 
  791         /*
  792          * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
  793          * (The absence of the TS_CREATE flag prevents the lwp from running
  794          * until we are finished with it, even if lwp_continue() is called on
  795          * it by some other lwp in the process or elsewhere in the kernel.)
  796          */
  797         thread_lock(t);
  798         ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
  799         /*
  800          * If TS_CSTART is set, lwp_continue(t) has been called and
  801          * has already incremented p_lwprcnt; avoid doing this twice.
  802          */
  803         if (!(t->t_schedflag & TS_CSTART))
  804                 p->p_lwprcnt++;
  805         t->t_schedflag |= (TS_CSTART | TS_CREATE);
  806         setrun_locked(t);
  807         thread_unlock(t);
  808 }
  809 
  810 /*
  811  * Copy an LWP's active templates, and clear the latest contracts.
  812  */
  813 void
  814 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
  815 {
  816         int i;
  817 
  818         for (i = 0; i < ct_ntypes; i++) {
  819                 dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
  820                 dst->lwp_ct_latest[i] = NULL;
  821         }
  822 }
  823 
  824 /*
  825  * Clear an LWP's contract template state.
  826  */
  827 void
  828 lwp_ctmpl_clear(klwp_t *lwp)
  829 {
  830         ct_template_t *tmpl;
  831         int i;
  832 
  833         for (i = 0; i < ct_ntypes; i++) {
  834                 if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
  835                         ctmpl_free(tmpl);
  836                         lwp->lwp_ct_active[i] = NULL;
  837                 }
  838 
  839                 if (lwp->lwp_ct_latest[i] != NULL) {
  840                         contract_rele(lwp->lwp_ct_latest[i]);
  841                         lwp->lwp_ct_latest[i] = NULL;
  842                 }
  843         }
  844 }
  845 
  846 /*
  847  * Individual lwp exit.
  848  * If this is the last lwp, exit the whole process.
  849  */
  850 void
  851 lwp_exit(void)
  852 {
  853         kthread_t *t = curthread;
  854         klwp_t *lwp = ttolwp(t);
  855         proc_t *p = ttoproc(t);
  856 
  857         ASSERT(MUTEX_HELD(&p->p_lock));
  858 
  859         mutex_exit(&p->p_lock);
  860 
  861 #if defined(__sparc)
  862         /*
  863          * Ensure that the user stack is fully abandoned..
  864          */
  865         trash_user_windows();
  866 #endif
  867 
  868         tsd_exit();                     /* free thread specific data */
  869 
  870         kcpc_passivate();               /* Clean up performance counter state */
  871 
  872         pollcleanup();
  873 
  874         if (t->t_door)
  875                 door_slam();
  876 
  877         if (t->t_schedctl != NULL)
  878                 schedctl_lwp_cleanup(t);
  879 
  880         if (t->t_upimutex != NULL)
  881                 upimutex_cleanup();
  882 
  883         /*
  884          * Perform any brand specific exit processing, then release any
  885          * brand data associated with the lwp
  886          */
  887         if (PROC_IS_BRANDED(p))
  888                 BROP(p)->b_lwpexit(lwp);
  889 
  890         lwp_pcb_exit();
  891 
  892         mutex_enter(&p->p_lock);
  893         lwp_cleanup();
  894 
  895         /*
  896          * When this process is dumping core, its lwps are held here
  897          * until the core dump is finished. Then exitlwps() is called
  898          * again to release these lwps so that they can finish exiting.
  899          */
  900         if (p->p_flag & SCOREDUMP)
  901                 stop(PR_SUSPENDED, SUSPEND_NORMAL);
  902 
  903         /*
  904          * Block the process against /proc now that we have really acquired
  905          * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
  906          */
  907         prbarrier(p);
  908 
  909         /*
  910          * Call proc_exit() if this is the last non-daemon lwp in the process.
  911          */
  912         if (!(t->t_proc_flag & TP_DAEMON) &&
  913             p->p_lwpcnt == p->p_lwpdaemon + 1) {
  914                 mutex_exit(&p->p_lock);
  915                 if (proc_exit(CLD_EXITED, 0) == 0) {
  916                         /* Restarting init. */
  917                         return;
  918                 }
  919 
  920                 /*
  921                  * proc_exit() returns a non-zero value when some other
  922                  * lwp got there first.  We just have to continue in
  923                  * lwp_exit().
  924                  */
  925                 mutex_enter(&p->p_lock);
  926                 ASSERT(curproc->p_flag & SEXITLWPS);
  927                 prbarrier(p);
  928         }
  929 
  930         DTRACE_PROC(lwp__exit);
  931 
  932         /*
  933          * If the lwp is a detached lwp or if the process is exiting,
  934          * remove (lwp_hash_out()) the lwp from the lwp directory.
  935          * Otherwise null out the lwp's le_thread pointer in the lwp
  936          * directory so that other threads will see it as a zombie lwp.
  937          */
  938         prlwpexit(t);           /* notify /proc */
  939         if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
  940                 lwp_hash_out(p, t->t_tid);
  941         else {
  942                 ASSERT(!(t->t_proc_flag & TP_DAEMON));
  943                 p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
  944                 p->p_zombcnt++;
  945                 cv_broadcast(&p->p_lwpexit);
  946         }
  947         if (t->t_proc_flag & TP_DAEMON) {
  948                 p->p_lwpdaemon--;
  949                 t->t_proc_flag &= ~TP_DAEMON;
  950         }
  951         t->t_proc_flag &= ~TP_TWAIT;
  952 
  953         /*
  954          * Maintain accurate lwp count for task.max-lwps resource control.
  955          */
  956         mutex_enter(&p->p_zone->zone_nlwps_lock);
  957         p->p_task->tk_nlwps--;
  958         p->p_task->tk_proj->kpj_nlwps--;
  959         p->p_zone->zone_nlwps--;
  960         mutex_exit(&p->p_zone->zone_nlwps_lock);
  961 
  962         CL_EXIT(t);             /* tell the scheduler that t is exiting */
  963         ASSERT(p->p_lwpcnt != 0);
  964         p->p_lwpcnt--;
  965 
  966         /*
  967          * If all remaining non-daemon lwps are waiting in lwp_wait(),
  968          * wake them up so someone can return EDEADLK.
  969          * (See the block comment preceeding lwp_wait().)
  970          */
  971         if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
  972                 cv_broadcast(&p->p_lwpexit);
  973 
  974         t->t_proc_flag |= TP_LWPEXIT;
  975         term_mstate(t);
  976 
  977 #ifndef NPROBE
  978         /* Kernel probe */
  979         if (t->t_tnf_tpdp)
  980                 tnf_thread_exit();
  981 #endif /* NPROBE */
  982 
  983         t->t_forw->t_back = t->t_back;
  984         t->t_back->t_forw = t->t_forw;
  985         if (t == p->p_tlist)
  986                 p->p_tlist = t->t_forw;
  987 
  988         /*
  989          * Clean up the signal state.
  990          */
  991         if (t->t_sigqueue != NULL)
  992                 sigdelq(p, t, 0);
  993         if (lwp->lwp_curinfo != NULL) {
  994                 siginfofree(lwp->lwp_curinfo);
  995                 lwp->lwp_curinfo = NULL;
  996         }
  997 
  998         thread_rele(t);
  999 
 1000         /*
 1001          * Terminated lwps are associated with process zero and are put onto
 1002          * death-row by resume().  Avoid preemption after resetting t->t_procp.
 1003          */
 1004         t->t_preempt++;
 1005 
 1006         if (t->t_ctx != NULL)
 1007                 exitctx(t);
 1008         if (p->p_pctx != NULL)
 1009                 exitpctx(p);
 1010 
 1011         t->t_procp = &p0;
 1012 
 1013         /*
 1014          * Notify the HAT about the change of address space
 1015          */
 1016         hat_thread_exit(t);
 1017         /*
 1018          * When this is the last running lwp in this process and some lwp is
 1019          * waiting for this condition to become true, or this thread was being
 1020          * suspended, then the waiting lwp is awakened.
 1021          *
 1022          * Also, if the process is exiting, we may have a thread waiting in
 1023          * exitlwps() that needs to be notified.
 1024          */
 1025         if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
 1026             (p->p_flag & SEXITLWPS))
 1027                 cv_broadcast(&p->p_holdlwps);
 1028 
 1029         /*
 1030          * Need to drop p_lock so we can reacquire pidlock.
 1031          */
 1032         mutex_exit(&p->p_lock);
 1033         mutex_enter(&pidlock);
 1034 
 1035         ASSERT(t != t->t_next);         /* t0 never exits */
 1036         t->t_next->t_prev = t->t_prev;
 1037         t->t_prev->t_next = t->t_next;
 1038         cv_broadcast(&t->t_joincv);     /* wake up anyone in thread_join */
 1039         mutex_exit(&pidlock);
 1040 
 1041         t->t_state = TS_ZOMB;
 1042         swtch_from_zombie();
 1043         /* never returns */
 1044 }
 1045 
 1046 
 1047 /*
 1048  * Cleanup function for an exiting lwp.
 1049  * Called both from lwp_exit() and from proc_exit().
 1050  * p->p_lock is repeatedly released and grabbed in this function.
 1051  */
 1052 void
 1053 lwp_cleanup(void)
 1054 {
 1055         kthread_t *t = curthread;
 1056         proc_t *p = ttoproc(t);
 1057 
 1058         ASSERT(MUTEX_HELD(&p->p_lock));
 1059 
 1060         /* untimeout any lwp-bound realtime timers */
 1061         if (p->p_itimer != NULL)
 1062                 timer_lwpexit();
 1063 
 1064         /*
 1065          * If this is the /proc agent lwp that is exiting, readjust p_lwpid
 1066          * so it appears that the agent never existed, and clear p_agenttp.
 1067          */
 1068         if (t == p->p_agenttp) {
 1069                 ASSERT(t->t_tid == p->p_lwpid);
 1070                 p->p_lwpid--;
 1071                 p->p_agenttp = NULL;
 1072         }
 1073 
 1074         /*
 1075          * Do lgroup bookkeeping to account for thread exiting.
 1076          */
 1077         kpreempt_disable();
 1078         lgrp_move_thread(t, NULL, 1);
 1079         if (t->t_tid == 1) {
 1080                 p->p_t1_lgrpid = LGRP_NONE;
 1081         }
 1082         kpreempt_enable();
 1083 
 1084         lwp_ctmpl_clear(ttolwp(t));
 1085 }
 1086 
 1087 int
 1088 lwp_suspend(kthread_t *t)
 1089 {
 1090         int tid;
 1091         proc_t *p = ttoproc(t);
 1092 
 1093         ASSERT(MUTEX_HELD(&p->p_lock));
 1094 
 1095         /*
 1096          * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
 1097          * If an lwp is stopping itself, there is no need to wait.
 1098          */
 1099 top:
 1100         t->t_proc_flag |= TP_HOLDLWP;
 1101         if (t == curthread) {
 1102                 t->t_sig_check = 1;
 1103         } else {
 1104                 /*
 1105                  * Make sure the lwp stops promptly.
 1106                  */
 1107                 thread_lock(t);
 1108                 t->t_sig_check = 1;
 1109                 /*
 1110                  * XXX Should use virtual stop like /proc does instead of
 1111                  * XXX waking the thread to get it to stop.
 1112                  */
 1113                 if (ISWAKEABLE(t) || ISWAITING(t)) {
 1114                         setrun_locked(t);
 1115                 } else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
 1116                         poke_cpu(t->t_cpu->cpu_id);
 1117                 }
 1118 
 1119                 tid = t->t_tid;  /* remember thread ID */
 1120                 /*
 1121                  * Wait for lwp to stop
 1122                  */
 1123                 while (!SUSPENDED(t)) {
 1124                         /*
 1125                          * Drop the thread lock before waiting and reacquire it
 1126                          * afterwards, so the thread can change its t_state
 1127                          * field.
 1128                          */
 1129                         thread_unlock(t);
 1130 
 1131                         /*
 1132                          * Check if aborted by exitlwps().
 1133                          */
 1134                         if (p->p_flag & SEXITLWPS)
 1135                                 lwp_exit();
 1136 
 1137                         /*
 1138                          * Cooperate with jobcontrol signals and /proc stopping
 1139                          * by calling cv_wait_sig() to wait for the target
 1140                          * lwp to stop.  Just using cv_wait() can lead to
 1141                          * deadlock because, if some other lwp has stopped
 1142                          * by either of these mechanisms, then p_lwprcnt will
 1143                          * never become zero if we do a cv_wait().
 1144                          */
 1145                         if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
 1146                                 return (EINTR);
 1147 
 1148                         /*
 1149                          * Check to see if thread died while we were
 1150                          * waiting for it to suspend.
 1151                          */
 1152                         if (idtot(p, tid) == NULL)
 1153                                 return (ESRCH);
 1154 
 1155                         thread_lock(t);
 1156                         /*
 1157                          * If the TP_HOLDLWP flag went away, lwp_continue()
 1158                          * or vfork() must have been called while we were
 1159                          * waiting, so start over again.
 1160                          */
 1161                         if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
 1162                                 thread_unlock(t);
 1163                                 goto top;
 1164                         }
 1165                 }
 1166                 thread_unlock(t);
 1167         }
 1168         return (0);
 1169 }
 1170 
 1171 /*
 1172  * continue a lwp that's been stopped by lwp_suspend().
 1173  */
 1174 void
 1175 lwp_continue(kthread_t *t)
 1176 {
 1177         proc_t *p = ttoproc(t);
 1178         int was_suspended = t->t_proc_flag & TP_HOLDLWP;
 1179 
 1180         ASSERT(MUTEX_HELD(&p->p_lock));
 1181 
 1182         t->t_proc_flag &= ~TP_HOLDLWP;
 1183         thread_lock(t);
 1184         if (SUSPENDED(t) &&
 1185             !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
 1186                 p->p_lwprcnt++;
 1187                 t->t_schedflag |= TS_CSTART;
 1188                 setrun_locked(t);
 1189         }
 1190         thread_unlock(t);
 1191         /*
 1192          * Wakeup anyone waiting for this thread to be suspended
 1193          */
 1194         if (was_suspended)
 1195                 cv_broadcast(&p->p_holdlwps);
 1196 }
 1197 
 1198 /*
 1199  * ********************************
 1200  *  Miscellaneous lwp routines    *
 1201  * ********************************
 1202  */
 1203 /*
 1204  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
 1205  * This will cause the process's lwps to stop at a hold point.  A hold
 1206  * point is where a kernel thread has a flat stack.  This is at the
 1207  * return from a system call and at the return from a user level trap.
 1208  *
 1209  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
 1210  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
 1211  * hold point.  The lwps in the process are not being cloned, so they
 1212  * are held at the usual hold points and also within issig_forreal().
 1213  * This has the side-effect that their system calls do not return
 1214  * showing EINTR.
 1215  *
 1216  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
 1217  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
 1218  * lwp is waiting for the target lwp to be stopped.
 1219  */
 1220 void
 1221 holdlwp(void)
 1222 {
 1223         proc_t *p = curproc;
 1224         kthread_t *t = curthread;
 1225 
 1226         mutex_enter(&p->p_lock);
 1227         /*
 1228          * Don't terminate immediately if the process is dumping core.
 1229          * Once the process has dumped core, all lwps are terminated.
 1230          */
 1231         if (!(p->p_flag & SCOREDUMP)) {
 1232                 if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
 1233                         lwp_exit();
 1234         }
 1235         if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
 1236                 mutex_exit(&p->p_lock);
 1237                 return;
 1238         }
 1239         /*
 1240          * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
 1241          * when p->p_lwprcnt becomes zero.
 1242          */
 1243         stop(PR_SUSPENDED, SUSPEND_NORMAL);
 1244         if (p->p_flag & SEXITLWPS)
 1245                 lwp_exit();
 1246         mutex_exit(&p->p_lock);
 1247 }
 1248 
 1249 /*
 1250  * Have all lwps within the process hold at a point where they are
 1251  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
 1252  */
 1253 int
 1254 holdlwps(int holdflag)
 1255 {
 1256         proc_t *p = curproc;
 1257 
 1258         ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
 1259         mutex_enter(&p->p_lock);
 1260         schedctl_finish_sigblock(curthread);
 1261 again:
 1262         while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
 1263                 /*
 1264                  * If another lwp is doing a forkall() or proc_exit(), bail out.
 1265                  */
 1266                 if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
 1267                         mutex_exit(&p->p_lock);
 1268                         return (0);
 1269                 }
 1270                 /*
 1271                  * Another lwp is doing a fork1() or is undergoing
 1272                  * watchpoint activity.  We hold here for it to complete.
 1273                  */
 1274                 stop(PR_SUSPENDED, SUSPEND_NORMAL);
 1275         }
 1276         p->p_flag |= holdflag;
 1277         pokelwps(p);
 1278         --p->p_lwprcnt;
 1279         /*
 1280          * Wait for the process to become quiescent (p->p_lwprcnt == 0).
 1281          */
 1282         while (p->p_lwprcnt > 0) {
 1283                 /*
 1284                  * Check if aborted by exitlwps().
 1285                  * Also check if SHOLDWATCH is set; it takes precedence.
 1286                  */
 1287                 if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
 1288                         p->p_lwprcnt++;
 1289                         p->p_flag &= ~holdflag;
 1290                         cv_broadcast(&p->p_holdlwps);
 1291                         goto again;
 1292                 }
 1293                 /*
 1294                  * Cooperate with jobcontrol signals and /proc stopping.
 1295                  * If some other lwp has stopped by either of these
 1296                  * mechanisms, then p_lwprcnt will never become zero
 1297                  * and the process will appear deadlocked unless we
 1298                  * stop here in sympathy with the other lwp before
 1299                  * doing the cv_wait() below.
 1300                  *
 1301                  * If the other lwp stops after we do the cv_wait(), it
 1302                  * will wake us up to loop around and do the sympathy stop.
 1303                  *
 1304                  * Since stop() drops p->p_lock, we must start from
 1305                  * the top again on returning from stop().
 1306                  */
 1307                 if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
 1308                         int whystop = p->p_stopsig? PR_JOBCONTROL :
 1309                             PR_REQUESTED;
 1310                         p->p_lwprcnt++;
 1311                         p->p_flag &= ~holdflag;
 1312                         stop(whystop, p->p_stopsig);
 1313                         goto again;
 1314                 }
 1315                 cv_wait(&p->p_holdlwps, &p->p_lock);
 1316         }
 1317         p->p_lwprcnt++;
 1318         p->p_flag &= ~holdflag;
 1319         mutex_exit(&p->p_lock);
 1320         return (1);
 1321 }
 1322 
 1323 /*
 1324  * See comments for holdwatch(), below.
 1325  */
 1326 static int
 1327 holdcheck(int clearflags)
 1328 {
 1329         proc_t *p = curproc;
 1330 
 1331         /*
 1332          * If we are trying to exit, that takes precedence over anything else.
 1333          */
 1334         if (p->p_flag & SEXITLWPS) {
 1335                 p->p_lwprcnt++;
 1336                 p->p_flag &= ~clearflags;
 1337                 lwp_exit();
 1338         }
 1339 
 1340         /*
 1341          * If another thread is calling fork1(), stop the current thread so the
 1342          * other can complete.
 1343          */
 1344         if (p->p_flag & SHOLDFORK1) {
 1345                 p->p_lwprcnt++;
 1346                 stop(PR_SUSPENDED, SUSPEND_NORMAL);
 1347                 if (p->p_flag & SEXITLWPS) {
 1348                         p->p_flag &= ~clearflags;
 1349                         lwp_exit();
 1350                 }
 1351                 return (-1);
 1352         }
 1353 
 1354         /*
 1355          * If another thread is calling fork(), then indicate we are doing
 1356          * watchpoint activity.  This will cause holdlwps() above to stop the
 1357          * forking thread, at which point we can continue with watchpoint
 1358          * activity.
 1359          */
 1360         if (p->p_flag & SHOLDFORK) {
 1361                 p->p_lwprcnt++;
 1362                 while (p->p_flag & SHOLDFORK) {
 1363                         p->p_flag |= SHOLDWATCH;
 1364                         cv_broadcast(&p->p_holdlwps);
 1365                         cv_wait(&p->p_holdlwps, &p->p_lock);
 1366                         p->p_flag &= ~SHOLDWATCH;
 1367                 }
 1368                 return (-1);
 1369         }
 1370 
 1371         return (0);
 1372 }
 1373 
 1374 /*
 1375  * Stop all lwps within the process, holding themselves in the kernel while the
 1376  * active lwp undergoes watchpoint activity.  This is more complicated than
 1377  * expected because stop() relies on calling holdwatch() in order to copyin data
 1378  * from the user's address space.  A double barrier is used to prevent an
 1379  * infinite loop.
 1380  *
 1381  *      o The first thread into holdwatch() is the 'master' thread and does
 1382  *        the following:
 1383  *
 1384  *              - Sets SHOLDWATCH on the current process
 1385  *              - Sets TP_WATCHSTOP on the current thread
 1386  *              - Waits for all threads to be either stopped or have
 1387  *                TP_WATCHSTOP set.
 1388  *              - Sets the SWATCHOK flag on the process
 1389  *              - Unsets TP_WATCHSTOP
 1390  *              - Waits for the other threads to completely stop
 1391  *              - Unsets SWATCHOK
 1392  *
 1393  *      o If SHOLDWATCH is already set when we enter this function, then another
 1394  *        thread is already trying to stop this thread.  This 'slave' thread
 1395  *        does the following:
 1396  *
 1397  *              - Sets TP_WATCHSTOP on the current thread
 1398  *              - Waits for SWATCHOK flag to be set
 1399  *              - Calls stop()
 1400  *
 1401  *      o If SWATCHOK is set on the process, then this function immediately
 1402  *        returns, as we must have been called via stop().
 1403  *
 1404  * In addition, there are other flags that take precedence over SHOLDWATCH:
 1405  *
 1406  *      o If SEXITLWPS is set, exit immediately.
 1407  *
 1408  *      o If SHOLDFORK1 is set, wait for fork1() to complete.
 1409  *
 1410  *      o If SHOLDFORK is set, then watchpoint activity takes precedence In this
 1411  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
 1412  *
 1413  *      o If the process is being stopped via /proc (TP_PRSTOP is set), then we
 1414  *        stop the current thread.
 1415  *
 1416  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
 1417  * threads were stopped, or the list of watched pages has changed.
 1418  */
 1419 int
 1420 holdwatch(void)
 1421 {
 1422         proc_t *p = curproc;
 1423         kthread_t *t = curthread;
 1424         int ret = 0;
 1425 
 1426         mutex_enter(&p->p_lock);
 1427 
 1428         p->p_lwprcnt--;
 1429 
 1430         /*
 1431          * Check for bail-out conditions as outlined above.
 1432          */
 1433         if (holdcheck(0) != 0) {
 1434                 mutex_exit(&p->p_lock);
 1435                 return (-1);
 1436         }
 1437 
 1438         if (!(p->p_flag & SHOLDWATCH)) {
 1439                 /*
 1440                  * We are the master watchpoint thread.  Set SHOLDWATCH and poke
 1441                  * the other threads.
 1442                  */
 1443                 p->p_flag |= SHOLDWATCH;
 1444                 pokelwps(p);
 1445 
 1446                 /*
 1447                  * Wait for all threads to be stopped or have TP_WATCHSTOP set.
 1448                  */
 1449                 while (pr_allstopped(p, 1) > 0) {
 1450                         if (holdcheck(SHOLDWATCH) != 0) {
 1451                                 p->p_flag &= ~SHOLDWATCH;
 1452                                 mutex_exit(&p->p_lock);
 1453                                 return (-1);
 1454                         }
 1455 
 1456                         cv_wait(&p->p_holdlwps, &p->p_lock);
 1457                 }
 1458 
 1459                 /*
 1460                  * All threads are now stopped or in the process of stopping.
 1461                  * Set SWATCHOK and let them stop completely.
 1462                  */
 1463                 p->p_flag |= SWATCHOK;
 1464                 t->t_proc_flag &= ~TP_WATCHSTOP;
 1465                 cv_broadcast(&p->p_holdlwps);
 1466 
 1467                 while (pr_allstopped(p, 0) > 0) {
 1468                         /*
 1469                          * At first glance, it may appear that we don't need a
 1470                          * call to holdcheck() here.  But if the process gets a
 1471                          * SIGKILL signal, one of our stopped threads may have
 1472                          * been awakened and is waiting in exitlwps(), which
 1473                          * takes precedence over watchpoints.
 1474                          */
 1475                         if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
 1476                                 p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
 1477                                 mutex_exit(&p->p_lock);
 1478                                 return (-1);
 1479                         }
 1480 
 1481                         cv_wait(&p->p_holdlwps, &p->p_lock);
 1482                 }
 1483 
 1484                 /*
 1485                  * All threads are now completely stopped.
 1486                  */
 1487                 p->p_flag &= ~SWATCHOK;
 1488                 p->p_flag &= ~SHOLDWATCH;
 1489                 p->p_lwprcnt++;
 1490 
 1491         } else if (!(p->p_flag & SWATCHOK)) {
 1492 
 1493                 /*
 1494                  * SHOLDWATCH is set, so another thread is trying to do
 1495                  * watchpoint activity.  Indicate this thread is stopping, and
 1496                  * wait for the OK from the master thread.
 1497                  */
 1498                 t->t_proc_flag |= TP_WATCHSTOP;
 1499                 cv_broadcast(&p->p_holdlwps);
 1500 
 1501                 while (!(p->p_flag & SWATCHOK)) {
 1502                         if (holdcheck(0) != 0) {
 1503                                 t->t_proc_flag &= ~TP_WATCHSTOP;
 1504                                 mutex_exit(&p->p_lock);
 1505                                 return (-1);
 1506                         }
 1507 
 1508                         cv_wait(&p->p_holdlwps, &p->p_lock);
 1509                 }
 1510 
 1511                 /*
 1512                  * Once the master thread has given the OK, this thread can
 1513                  * actually call stop().
 1514                  */
 1515                 t->t_proc_flag &= ~TP_WATCHSTOP;
 1516                 p->p_lwprcnt++;
 1517 
 1518                 stop(PR_SUSPENDED, SUSPEND_NORMAL);
 1519 
 1520                 /*
 1521                  * It's not OK to do watchpoint activity, notify caller to
 1522                  * retry.
 1523                  */
 1524                 ret = -1;
 1525 
 1526         } else {
 1527 
 1528                 /*
 1529                  * The only way we can hit the case where SHOLDWATCH is set and
 1530                  * SWATCHOK is set is if we are triggering this from within a
 1531                  * stop() call.  Assert that this is the case.
 1532                  */
 1533 
 1534                 ASSERT(t->t_proc_flag & TP_STOPPING);
 1535                 p->p_lwprcnt++;
 1536         }
 1537 
 1538         mutex_exit(&p->p_lock);
 1539 
 1540         return (ret);
 1541 }
 1542 
 1543 /*
 1544  * force all interruptible lwps to trap into the kernel.
 1545  */
 1546 void
 1547 pokelwps(proc_t *p)
 1548 {
 1549         kthread_t *t;
 1550 
 1551         ASSERT(MUTEX_HELD(&p->p_lock));
 1552 
 1553         t = p->p_tlist;
 1554         do {
 1555                 if (t == curthread)
 1556                         continue;
 1557                 thread_lock(t);
 1558                 aston(t);       /* make thread trap or do post_syscall */
 1559                 if (ISWAKEABLE(t) || ISWAITING(t)) {
 1560                         setrun_locked(t);
 1561                 } else if (t->t_state == TS_STOPPED) {
 1562                         /*
 1563                          * Ensure that proc_exit() is not blocked by lwps
 1564                          * that were stopped via jobcontrol or /proc.
 1565                          */
 1566                         if (p->p_flag & SEXITLWPS) {
 1567                                 p->p_stopsig = 0;
 1568                                 t->t_schedflag |= (TS_XSTART | TS_PSTART);
 1569                                 setrun_locked(t);
 1570                         }
 1571                         /*
 1572                          * If we are holding lwps for a forkall(),
 1573                          * force lwps that have been suspended via
 1574                          * lwp_suspend() and are suspended inside
 1575                          * of a system call to proceed to their
 1576                          * holdlwp() points where they are clonable.
 1577                          */
 1578                         if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
 1579                                 if ((t->t_schedflag & TS_CSTART) == 0) {
 1580                                         p->p_lwprcnt++;
 1581                                         t->t_schedflag |= TS_CSTART;
 1582                                         setrun_locked(t);
 1583                                 }
 1584                         }
 1585                 } else if (t->t_state == TS_ONPROC) {
 1586                         if (t->t_cpu != CPU)
 1587                                 poke_cpu(t->t_cpu->cpu_id);
 1588                 }
 1589                 thread_unlock(t);
 1590         } while ((t = t->t_forw) != p->p_tlist);
 1591 }
 1592 
 1593 /*
 1594  * undo the effects of holdlwps() or holdwatch().
 1595  */
 1596 void
 1597 continuelwps(proc_t *p)
 1598 {
 1599         kthread_t *t;
 1600 
 1601         /*
 1602          * If this flag is set, then the original holdwatch() didn't actually
 1603          * stop the process.  See comments for holdwatch().
 1604          */
 1605         if (p->p_flag & SWATCHOK) {
 1606                 ASSERT(curthread->t_proc_flag & TP_STOPPING);
 1607                 return;
 1608         }
 1609 
 1610         ASSERT(MUTEX_HELD(&p->p_lock));
 1611         ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
 1612 
 1613         t = p->p_tlist;
 1614         do {
 1615                 thread_lock(t);         /* SUSPENDED looks at t_schedflag */
 1616                 if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
 1617                         p->p_lwprcnt++;
 1618                         t->t_schedflag |= TS_CSTART;
 1619                         setrun_locked(t);
 1620                 }
 1621                 thread_unlock(t);
 1622         } while ((t = t->t_forw) != p->p_tlist);
 1623 }
 1624 
 1625 /*
 1626  * Force all other LWPs in the current process other than the caller to exit,
 1627  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
 1628  * is typically used in these situations:
 1629  *
 1630  *   (a) prior to an exec() system call
 1631  *   (b) prior to dumping a core file
 1632  *   (c) prior to a uadmin() shutdown
 1633  *
 1634  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
 1635  * Multiple threads in the process can call this function at one time by
 1636  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
 1637  * to declare one particular thread the winner who gets to kill the others.
 1638  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
 1639  * appropriate errno value is returned to caller for its system call to return.
 1640  */
 1641 int
 1642 exitlwps(int coredump)
 1643 {
 1644         proc_t *p = curproc;
 1645         int heldcnt;
 1646 
 1647         if (curthread->t_door)
 1648                 door_slam();
 1649         if (p->p_door_list)
 1650                 door_revoke_all();
 1651         if (curthread->t_schedctl != NULL)
 1652                 schedctl_lwp_cleanup(curthread);
 1653 
 1654         /*
 1655          * Ensure that before starting to wait for other lwps to exit,
 1656          * cleanup all upimutexes held by curthread. Otherwise, some other
 1657          * lwp could be waiting (uninterruptibly) for a upimutex held by
 1658          * curthread, and the call to pokelwps() below would deadlock.
 1659          * Even if a blocked upimutex_lock is made interruptible,
 1660          * curthread's upimutexes need to be unlocked: do it here.
 1661          */
 1662         if (curthread->t_upimutex != NULL)
 1663                 upimutex_cleanup();
 1664 
 1665         /*
 1666          * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
 1667          * We must also block any further /proc access from this point forward.
 1668          */
 1669         mutex_enter(&p->p_lock);
 1670         prbarrier(p);
 1671 
 1672         if (p->p_flag & SEXITLWPS) {
 1673                 mutex_exit(&p->p_lock);
 1674                 aston(curthread);       /* force a trip through post_syscall */
 1675                 return (set_errno(EINTR));
 1676         }
 1677 
 1678         p->p_flag |= SEXITLWPS;
 1679         if (coredump)           /* tell other lwps to stop, not exit */
 1680                 p->p_flag |= SCOREDUMP;
 1681 
 1682         /*
 1683          * Give precedence to exitlwps() if a holdlwps() is
 1684          * in progress. The lwp doing the holdlwps() operation
 1685          * is aborted when it is awakened.
 1686          */
 1687         while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
 1688                 cv_broadcast(&p->p_holdlwps);
 1689                 cv_wait(&p->p_holdlwps, &p->p_lock);
 1690                 prbarrier(p);
 1691         }
 1692         p->p_flag |= SHOLDFORK;
 1693         pokelwps(p);
 1694 
 1695         /*
 1696          * Wait for process to become quiescent.
 1697          */
 1698         --p->p_lwprcnt;
 1699         while (p->p_lwprcnt > 0) {
 1700                 cv_wait(&p->p_holdlwps, &p->p_lock);
 1701                 prbarrier(p);
 1702         }
 1703         p->p_lwprcnt++;
 1704         ASSERT(p->p_lwprcnt == 1);
 1705 
 1706         /*
 1707          * The SCOREDUMP flag puts the process into a quiescent
 1708          * state.  The process's lwps remain attached to this
 1709          * process until exitlwps() is called again without the
 1710          * 'coredump' flag set, then the lwps are terminated
 1711          * and the process can exit.
 1712          */
 1713         if (coredump) {
 1714                 p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
 1715                 goto out;
 1716         }
 1717 
 1718         /*
 1719          * Determine if there are any lwps left dangling in
 1720          * the stopped state.  This happens when exitlwps()
 1721          * aborts a holdlwps() operation.
 1722          */
 1723         p->p_flag &= ~SHOLDFORK;
 1724         if ((heldcnt = p->p_lwpcnt) > 1) {
 1725                 kthread_t *t;
 1726                 for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
 1727                         t->t_proc_flag &= ~TP_TWAIT;
 1728                         lwp_continue(t);
 1729                 }
 1730         }
 1731 
 1732         /*
 1733          * Wait for all other lwps to exit.
 1734          */
 1735         --p->p_lwprcnt;
 1736         while (p->p_lwpcnt > 1) {
 1737                 cv_wait(&p->p_holdlwps, &p->p_lock);
 1738                 prbarrier(p);
 1739         }
 1740         ++p->p_lwprcnt;
 1741         ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
 1742 
 1743         p->p_flag &= ~SEXITLWPS;
 1744         curthread->t_proc_flag &= ~TP_TWAIT;
 1745 
 1746 out:
 1747         if (!coredump && p->p_zombcnt) {        /* cleanup the zombie lwps */
 1748                 lwpdir_t *ldp;
 1749                 lwpent_t *lep;
 1750                 int i;
 1751 
 1752                 for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
 1753                         lep = ldp->ld_entry;
 1754                         if (lep != NULL && lep->le_thread != curthread) {
 1755                                 ASSERT(lep->le_thread == NULL);
 1756                                 p->p_zombcnt--;
 1757                                 lwp_hash_out(p, lep->le_lwpid);
 1758                         }
 1759                 }
 1760                 ASSERT(p->p_zombcnt == 0);
 1761         }
 1762 
 1763         /*
 1764          * If some other LWP in the process wanted us to suspend ourself,
 1765          * then we will not do it.  The other LWP is now terminated and
 1766          * no one will ever continue us again if we suspend ourself.
 1767          */
 1768         curthread->t_proc_flag &= ~TP_HOLDLWP;
 1769         p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
 1770         mutex_exit(&p->p_lock);
 1771         return (0);
 1772 }
 1773 
 1774 /*
 1775  * duplicate a lwp.
 1776  */
 1777 klwp_t *
 1778 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
 1779 {
 1780         klwp_t *clwp;
 1781         void *tregs, *tfpu;
 1782         kthread_t *t = lwptot(lwp);
 1783         kthread_t *ct;
 1784         proc_t *p = lwptoproc(lwp);
 1785         int cid;
 1786         void *bufp;
 1787         void *brand_data;
 1788         int val;
 1789 
 1790         ASSERT(p == curproc);
 1791         ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
 1792 
 1793 #if defined(__sparc)
 1794         if (t == curthread)
 1795                 (void) flush_user_windows_to_stack(NULL);
 1796 #endif
 1797 
 1798         if (t == curthread)
 1799                 /* copy args out of registers first */
 1800                 (void) save_syscall_args();
 1801 
 1802         clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
 1803             NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
 1804         if (clwp == NULL)
 1805                 return (NULL);
 1806 
 1807         /*
 1808          * most of the parent's lwp can be copied to its duplicate,
 1809          * except for the fields that are unique to each lwp, like
 1810          * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
 1811          */
 1812         ct = clwp->lwp_thread;
 1813         tregs = clwp->lwp_regs;
 1814         tfpu = clwp->lwp_fpu;
 1815         brand_data = clwp->lwp_brand;
 1816 
 1817         /*
 1818          * Copy parent lwp to child lwp.  Hold child's p_lock to prevent
 1819          * mstate_aggr_state() from reading stale mstate entries copied
 1820          * from lwp to clwp.
 1821          */
 1822         mutex_enter(&cp->p_lock);
 1823         *clwp = *lwp;
 1824 
 1825         /* clear microstate and resource usage data in new lwp */
 1826         init_mstate(ct, LMS_STOPPED);
 1827         bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
 1828         mutex_exit(&cp->p_lock);
 1829 
 1830         /* fix up child's lwp */
 1831 
 1832         clwp->lwp_pcb.pcb_flags = 0;
 1833 #if defined(__sparc)
 1834         clwp->lwp_pcb.pcb_step = STEP_NONE;
 1835 #endif
 1836         clwp->lwp_cursig = 0;
 1837         clwp->lwp_extsig = 0;
 1838         clwp->lwp_curinfo = (struct sigqueue *)0;
 1839         clwp->lwp_thread = ct;
 1840         ct->t_sysnum = t->t_sysnum;
 1841         clwp->lwp_regs = tregs;
 1842         clwp->lwp_fpu = tfpu;
 1843         clwp->lwp_brand = brand_data;
 1844         clwp->lwp_ap = clwp->lwp_arg;
 1845         clwp->lwp_procp = cp;
 1846         bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
 1847         clwp->lwp_lastfault = 0;
 1848         clwp->lwp_lastfaddr = 0;
 1849 
 1850         /* copy parent's struct regs to child. */
 1851         lwp_forkregs(lwp, clwp);
 1852 
 1853         /*
 1854          * Fork thread context ops, if any.
 1855          */
 1856         if (t->t_ctx)
 1857                 forkctx(t, ct);
 1858 
 1859         /* fix door state in the child */
 1860         if (t->t_door)
 1861                 door_fork(t, ct);
 1862 
 1863         /* copy current contract templates, clear latest contracts */
 1864         lwp_ctmpl_copy(clwp, lwp);
 1865 
 1866         mutex_enter(&cp->p_lock);
 1867         /* lwp_create() set the TP_HOLDLWP flag */
 1868         if (!(t->t_proc_flag & TP_HOLDLWP))
 1869                 ct->t_proc_flag &= ~TP_HOLDLWP;
 1870         if (cp->p_flag & SMSACCT)
 1871                 ct->t_proc_flag |= TP_MSACCT;
 1872         mutex_exit(&cp->p_lock);
 1873 
 1874         /* Allow brand to propagate brand-specific state */
 1875         if (PROC_IS_BRANDED(p))
 1876                 BROP(p)->b_forklwp(lwp, clwp);
 1877 
 1878 retry:
 1879         cid = t->t_cid;
 1880 
 1881         val = CL_ALLOC(&bufp, cid, KM_SLEEP);
 1882         ASSERT(val == 0);
 1883 
 1884         mutex_enter(&p->p_lock);
 1885         if (cid != t->t_cid) {
 1886                 /*
 1887                  * Someone just changed this thread's scheduling class,
 1888                  * so try pre-allocating the buffer again.  Hopefully we
 1889                  * don't hit this often.
 1890                  */
 1891                 mutex_exit(&p->p_lock);
 1892                 CL_FREE(cid, bufp);
 1893                 goto retry;
 1894         }
 1895 
 1896         ct->t_unpark = t->t_unpark;
 1897         ct->t_clfuncs = t->t_clfuncs;
 1898         CL_FORK(t, ct, bufp);
 1899         ct->t_cid = t->t_cid;   /* after data allocated so prgetpsinfo works */
 1900         mutex_exit(&p->p_lock);
 1901 
 1902         return (clwp);
 1903 }
 1904 
 1905 /*
 1906  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
 1907  */
 1908 void
 1909 lwp_hash_in(proc_t *p, lwpent_t *lep, tidhash_t *tidhash, uint_t tidhash_sz,
 1910     int do_lock)
 1911 {
 1912         tidhash_t *thp = &tidhash[TIDHASH(lep->le_lwpid, tidhash_sz)];
 1913         lwpdir_t **ldpp;
 1914         lwpdir_t *ldp;
 1915         kthread_t *t;
 1916 
 1917         /*
 1918          * Allocate a directory element from the free list.
 1919          * Code elsewhere guarantees a free slot.
 1920          */
 1921         ldp = p->p_lwpfree;
 1922         p->p_lwpfree = ldp->ld_next;
 1923         ASSERT(ldp->ld_entry == NULL);
 1924         ldp->ld_entry = lep;
 1925 
 1926         if (do_lock)
 1927                 mutex_enter(&thp->th_lock);
 1928 
 1929         /*
 1930          * Insert it into the lwpid hash table.
 1931          */
 1932         ldpp = &thp->th_list;
 1933         ldp->ld_next = *ldpp;
 1934         *ldpp = ldp;
 1935 
 1936         /*
 1937          * Set the active thread's directory slot entry.
 1938          */
 1939         if ((t = lep->le_thread) != NULL) {
 1940                 ASSERT(lep->le_lwpid == t->t_tid);
 1941                 t->t_dslot = (int)(ldp - p->p_lwpdir);
 1942         }
 1943 
 1944         if (do_lock)
 1945                 mutex_exit(&thp->th_lock);
 1946 }
 1947 
 1948 /*
 1949  * Remove an lwp from the lwpid hash table and free its directory entry.
 1950  * This is done when a detached lwp exits in lwp_exit() or
 1951  * when a non-detached lwp is waited for in lwp_wait() or
 1952  * when a zombie lwp is detached in lwp_detach().
 1953  */
 1954 void
 1955 lwp_hash_out(proc_t *p, id_t lwpid)
 1956 {
 1957         tidhash_t *thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
 1958         lwpdir_t **ldpp;
 1959         lwpdir_t *ldp;
 1960         lwpent_t *lep;
 1961 
 1962         mutex_enter(&thp->th_lock);
 1963         for (ldpp = &thp->th_list;
 1964             (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
 1965                 lep = ldp->ld_entry;
 1966                 if (lep->le_lwpid == lwpid) {
 1967                         prlwpfree(p, lep);      /* /proc deals with le_trace */
 1968                         *ldpp = ldp->ld_next;
 1969                         ldp->ld_entry = NULL;
 1970                         ldp->ld_next = p->p_lwpfree;
 1971                         p->p_lwpfree = ldp;
 1972                         kmem_free(lep, sizeof (*lep));
 1973                         break;
 1974                 }
 1975         }
 1976         mutex_exit(&thp->th_lock);
 1977 }
 1978 
 1979 /*
 1980  * Lookup an lwp in the lwpid hash table by lwpid.
 1981  */
 1982 lwpdir_t *
 1983 lwp_hash_lookup(proc_t *p, id_t lwpid)
 1984 {
 1985         tidhash_t *thp;
 1986         lwpdir_t *ldp;
 1987 
 1988         /*
 1989          * The process may be exiting, after p_tidhash has been set to NULL in
 1990          * proc_exit() but before prfee() has been called.  Return failure in
 1991          * this case.
 1992          */
 1993         if (p->p_tidhash == NULL)
 1994                 return (NULL);
 1995 
 1996         thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
 1997         for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
 1998                 if (ldp->ld_entry->le_lwpid == lwpid)
 1999                         return (ldp);
 2000         }
 2001 
 2002         return (NULL);
 2003 }
 2004 
 2005 /*
 2006  * Same as lwp_hash_lookup(), but acquire and return
 2007  * the tid hash table entry lock on success.
 2008  */
 2009 lwpdir_t *
 2010 lwp_hash_lookup_and_lock(proc_t *p, id_t lwpid, kmutex_t **mpp)
 2011 {
 2012         tidhash_t *tidhash;
 2013         uint_t tidhash_sz;
 2014         tidhash_t *thp;
 2015         lwpdir_t *ldp;
 2016 
 2017 top:
 2018         tidhash_sz = p->p_tidhash_sz;
 2019         membar_consumer();
 2020         if ((tidhash = p->p_tidhash) == NULL)
 2021                 return (NULL);
 2022 
 2023         thp = &tidhash[TIDHASH(lwpid, tidhash_sz)];
 2024         mutex_enter(&thp->th_lock);
 2025 
 2026         /*
 2027          * Since we are not holding p->p_lock, the tid hash table
 2028          * may have changed.  If so, start over.  If not, then
 2029          * it cannot change until after we drop &thp->th_lock;
 2030          */
 2031         if (tidhash != p->p_tidhash || tidhash_sz != p->p_tidhash_sz) {
 2032                 mutex_exit(&thp->th_lock);
 2033                 goto top;
 2034         }
 2035 
 2036         for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
 2037                 if (ldp->ld_entry->le_lwpid == lwpid) {
 2038                         *mpp = &thp->th_lock;
 2039                         return (ldp);
 2040                 }
 2041         }
 2042 
 2043         mutex_exit(&thp->th_lock);
 2044         return (NULL);
 2045 }
 2046 
 2047 /*
 2048  * Update the indicated LWP usage statistic for the current LWP.
 2049  */
 2050 void
 2051 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
 2052 {
 2053         klwp_t *lwp = ttolwp(curthread);
 2054 
 2055         if (lwp == NULL)
 2056                 return;
 2057 
 2058         switch (lwp_stat_id) {
 2059         case LWP_STAT_INBLK:
 2060                 lwp->lwp_ru.inblock += inc;
 2061                 break;
 2062         case LWP_STAT_OUBLK:
 2063                 lwp->lwp_ru.oublock += inc;
 2064                 break;
 2065         case LWP_STAT_MSGRCV:
 2066                 lwp->lwp_ru.msgrcv += inc;
 2067                 break;
 2068         case LWP_STAT_MSGSND:
 2069                 lwp->lwp_ru.msgsnd += inc;
 2070                 break;
 2071         default:
 2072                 panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
 2073         }
 2074 }

Cache object: 6cbd7417ca31a6ff3326b92b905acc12


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.