FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_lwp.c

     /*      $NetBSD: kern_lwp.c,v 1.251 2022/07/01 01:06:04 riastradh Exp $ */
     
     /*-
      * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020
      *     The NetBSD Foundation, Inc.
      * All rights reserved.
      *
      * This code is derived from software contributed to The NetBSD Foundation
      * by Nathan J. Williams, and Andrew Doran.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     *
     * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     * POSSIBILITY OF SUCH DAMAGE.
     */
    
    /*
     * Overview
     *
     *      Lightweight processes (LWPs) are the basic unit or thread of
     *      execution within the kernel.  The core state of an LWP is described
     *      by "struct lwp", also known as lwp_t.
     *
     *      Each LWP is contained within a process (described by "struct proc"),
     *      Every process contains at least one LWP, but may contain more.  The
     *      process describes attributes shared among all of its LWPs such as a
     *      private address space, global execution state (stopped, active,
     *      zombie, ...), signal disposition and so on.  On a multiprocessor
     *      machine, multiple LWPs be executing concurrently in the kernel.
     *
     * Execution states
     *
     *      At any given time, an LWP has overall state that is described by
     *      lwp::l_stat.  The states are broken into two sets below.  The first
     *      set is guaranteed to represent the absolute, current state of the
     *      LWP:
     *
     *      LSONPROC
     *
     *              On processor: the LWP is executing on a CPU, either in the
     *              kernel or in user space.
     *
     *      LSRUN
     *
     *              Runnable: the LWP is parked on a run queue, and may soon be
     *              chosen to run by an idle processor, or by a processor that
     *              has been asked to preempt a currently runnning but lower
     *              priority LWP.
     *
     *      LSIDL
     *
     *              Idle: the LWP has been created but has not yet executed, or
     *              it has ceased executing a unit of work and is waiting to be
     *              started again.  This state exists so that the LWP can occupy
     *              a slot in the process & PID table, but without having to
     *              worry about being touched; lookups of the LWP by ID will
     *              fail while in this state.  The LWP will become visible for
     *              lookup once its state transitions further.  Some special
     *              kernel threads also (ab)use this state to indicate that they
     *              are idle (soft interrupts and idle LWPs).
     *
     *      LSSUSPENDED:
     *
     *              Suspended: the LWP has had its execution suspended by
     *              another LWP in the same process using the _lwp_suspend()
     *              system call.  User-level LWPs also enter the suspended
     *              state when the system is shutting down.
     *
     *      The second set represent a "statement of intent" on behalf of the
     *      LWP.  The LWP may in fact be executing on a processor, may be
     *      sleeping or idle. It is expected to take the necessary action to
     *      stop executing or become "running" again within a short timeframe.
     *      The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
     *      Importantly, it indicates that its state is tied to a CPU.
     *
     *      LSZOMB:
     *
     *              Dead or dying: the LWP has released most of its resources
     *              and is about to switch away into oblivion, or has already
     *              switched away.  When it switches away, its few remaining
     *              resources can be collected.
     *
     *      LSSLEEP:
    *
    *              Sleeping: the LWP has entered itself onto a sleep queue, and
    *              has switched away or will switch away shortly to allow other
    *              LWPs to run on the CPU.
    *
    *      LSSTOP:
    *
    *              Stopped: the LWP has been stopped as a result of a job
    *              control signal, or as a result of the ptrace() interface. 
    *
    *              Stopped LWPs may run briefly within the kernel to handle
    *              signals that they receive, but will not return to user space
    *              until their process' state is changed away from stopped. 
    *
    *              Single LWPs within a process can not be set stopped
    *              selectively: all actions that can stop or continue LWPs
    *              occur at the process level.
    *
    * State transitions
    *
    *      Note that the LSSTOP state may only be set when returning to
    *      user space in userret(), or when sleeping interruptably.  The
    *      LSSUSPENDED state may only be set in userret().  Before setting
    *      those states, we try to ensure that the LWPs will release all
    *      locks that they hold, and at a minimum try to ensure that the
    *      LWP can be set runnable again by a signal.
    *
    *      LWPs may transition states in the following ways:
    *
    *       RUN -------> ONPROC            ONPROC -----> RUN
    *                                                  > SLEEP
    *                                                  > STOPPED
    *                                                  > SUSPENDED
    *                                                  > ZOMB
    *                                                  > IDL (special cases)
    *
    *       STOPPED ---> RUN               SUSPENDED --> RUN
    *                  > SLEEP
    *
    *       SLEEP -----> ONPROC            IDL --------> RUN
    *                  > RUN                           > SUSPENDED
    *                  > STOPPED                       > STOPPED
    *                                                  > ONPROC (special cases)
    *
    *      Some state transitions are only possible with kernel threads (eg
    *      ONPROC -> IDL) and happen under tightly controlled circumstances
    *      free of unwanted side effects.
    *
    * Migration
    *
    *      Migration of threads from one CPU to another could be performed
    *      internally by the scheduler via sched_takecpu() or sched_catchlwp()
    *      functions.  The universal lwp_migrate() function should be used for
    *      any other cases.  Subsystems in the kernel must be aware that CPU
    *      of LWP may change, while it is not locked.
    *
    * Locking
    *
    *      The majority of fields in 'struct lwp' are covered by a single,
    *      general spin lock pointed to by lwp::l_mutex.  The locks covering
    *      each field are documented in sys/lwp.h.
    *
    *      State transitions must be made with the LWP's general lock held,
    *      and may cause the LWP's lock pointer to change.  Manipulation of
    *      the general lock is not performed directly, but through calls to
    *      lwp_lock(), lwp_unlock() and others.  It should be noted that the
    *      adaptive locks are not allowed to be released while the LWP's lock
    *      is being held (unlike for other spin-locks).
    *
    *      States and their associated locks:
    *
    *      LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
    *
    *              Always covered by spc_lwplock, which protects LWPs not
    *              associated with any other sync object.  This is a per-CPU
    *              lock and matches lwp::l_cpu.
    *
    *      LSRUN:
    *
    *              Always covered by spc_mutex, which protects the run queues.
    *              This is a per-CPU lock and matches lwp::l_cpu.
    *
    *      LSSLEEP:
    *
    *              Covered by a lock associated with the sleep queue (sometimes
    *              a turnstile sleep queue) that the LWP resides on.  This can
    *              be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
    *
    *      LSSTOP:
    *
    *              If the LWP was previously sleeping (l_wchan != NULL), then
    *              l_mutex references the sleep queue lock.  If the LWP was
    *              runnable or on the CPU when halted, or has been removed from
    *              the sleep queue since halted, then the lock is spc_lwplock.
    *
    *      The lock order is as follows:
    *
    *              sleepq -> turnstile -> spc_lwplock -> spc_mutex
    *
    *      Each process has a scheduler state lock (proc::p_lock), and a
    *      number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    *      so on.  When an LWP is to be entered into or removed from one of the
    *      following states, p_lock must be held and the process wide counters
    *      adjusted:
    *
    *              LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    *
    *      (But not always for kernel threads.  There are some special cases
    *      as mentioned above: soft interrupts, and the idle loops.)
    *
    *      Note that an LWP is considered running or likely to run soon if in
    *      one of the following states.  This affects the value of p_nrlwps:
    *
    *              LSRUN, LSONPROC, LSSLEEP
    *
    *      p_lock does not need to be held when transitioning among these
    *      three states, hence p_lock is rarely taken for state transitions.
    */
   
   #include <sys/cdefs.h>
   __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.251 2022/07/01 01:06:04 riastradh Exp $");
   
   #include "opt_ddb.h"
   #include "opt_lockdebug.h"
   #include "opt_dtrace.h"
   
   #define _LWP_API_PRIVATE
   
   #include <sys/param.h>
   #include <sys/systm.h>
   #include <sys/cpu.h>
   #include <sys/pool.h>
   #include <sys/proc.h>
   #include <sys/syscallargs.h>
   #include <sys/syscall_stats.h>
   #include <sys/kauth.h>
   #include <sys/sleepq.h>
   #include <sys/lockdebug.h>
   #include <sys/kmem.h>
   #include <sys/pset.h>
   #include <sys/intr.h>
   #include <sys/lwpctl.h>
   #include <sys/atomic.h>
   #include <sys/filedesc.h>
   #include <sys/fstrans.h>
   #include <sys/dtrace_bsd.h>
   #include <sys/sdt.h>
   #include <sys/ptrace.h>
   #include <sys/xcall.h>
   #include <sys/uidinfo.h>
   #include <sys/sysctl.h>
   #include <sys/psref.h>
   #include <sys/msan.h>
   #include <sys/kcov.h>
   #include <sys/cprng.h>
   #include <sys/futex.h>
   
   #include <uvm/uvm_extern.h>
   #include <uvm/uvm_object.h>
   
   static pool_cache_t     lwp_cache       __read_mostly;
   struct lwplist          alllwp          __cacheline_aligned;
   
   static int              lwp_ctor(void *, void *, int);
   static void             lwp_dtor(void *, void *);
   
   /* DTrace proc provider probes */
   SDT_PROVIDER_DEFINE(proc);
   
   SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
   SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
   SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
   
   struct turnstile turnstile0 __cacheline_aligned;
   struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
   #ifdef LWP0_CPU_INFO
           .l_cpu = LWP0_CPU_INFO,
   #endif
   #ifdef LWP0_MD_INITIALIZER
           .l_md = LWP0_MD_INITIALIZER,
   #endif
           .l_proc = &proc0,
           .l_lid = 0,             /* we own proc0's slot in the pid table */
           .l_flag = LW_SYSTEM,
           .l_stat = LSONPROC,
           .l_ts = &turnstile0,
           .l_syncobj = &sched_syncobj,
           .l_refcnt = 0,
           .l_priority = PRI_USER + NPRI_USER - 1,
           .l_inheritedprio = -1,
           .l_class = SCHED_OTHER,
           .l_psid = PS_NONE,
           .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
           .l_name = __UNCONST("swapper"),
           .l_fd = &filedesc0,
   };
   
   static int
   lwp_maxlwp(void)
   {
           /* Assume 1 LWP per 1MiB. */
           uint64_t lwps_per = ctob(physmem) / (1024 * 1024);
   
           return MAX(MIN(MAXMAXLWP, lwps_per), MAXLWP);
   }
   
   static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
   
   /*
    * sysctl helper routine for kern.maxlwp. Ensures that the new
    * values are not too low or too high.
    */
   static int
   sysctl_kern_maxlwp(SYSCTLFN_ARGS)
   {
           int error, nmaxlwp;
           struct sysctlnode node;
   
           nmaxlwp = maxlwp;
           node = *rnode;
           node.sysctl_data = &nmaxlwp;
           error = sysctl_lookup(SYSCTLFN_CALL(&node));
           if (error || newp == NULL)
                   return error;
   
           if (nmaxlwp < 0 || nmaxlwp >= MAXMAXLWP)
                   return EINVAL;
           if (nmaxlwp > lwp_maxlwp())
                   return EINVAL;
           maxlwp = nmaxlwp;
   
           return 0;
   }
   
   static void
   sysctl_kern_lwp_setup(void)
   {
           sysctl_createv(NULL, 0, NULL, NULL,
                          CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
                          CTLTYPE_INT, "maxlwp",
                          SYSCTL_DESCR("Maximum number of simultaneous threads"),
                          sysctl_kern_maxlwp, 0, NULL, 0,
                          CTL_KERN, CTL_CREATE, CTL_EOL);
   }
   
   void
   lwpinit(void)
   {
   
           LIST_INIT(&alllwp);
           lwpinit_specificdata();
           /*
            * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
            * calls will exit before memory of LWPs is returned to the pool, where
            * KVA of LWP structure might be freed and re-used for other purposes.
            * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
            * callers, therefore a regular passive serialization barrier will
            * do the job.
            */
           lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0,
               PR_PSERIALIZE, "lwppl", NULL, IPL_NONE, lwp_ctor, lwp_dtor, NULL);
   
           maxlwp = lwp_maxlwp();
           sysctl_kern_lwp_setup();
   }
   
   void
   lwp0_init(void)
   {
           struct lwp *l = &lwp0;
   
           KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
   
           LIST_INSERT_HEAD(&alllwp, l, l_list);
   
           callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
           callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
           cv_init(&l->l_sigcv, "sigwait");
           cv_init(&l->l_waitcv, "vfork");
   
           kauth_cred_hold(proc0.p_cred);
           l->l_cred = proc0.p_cred;
   
           kdtrace_thread_ctor(NULL, l);
           lwp_initspecific(l);
   
           SYSCALL_TIME_LWP_INIT(l);
   }
   
   /*
    * Initialize the non-zeroed portion of an lwp_t.
    */
   static int
   lwp_ctor(void *arg, void *obj, int flags)
   {
           lwp_t *l = obj;
   
           l->l_stat = LSIDL;
           l->l_cpu = curcpu();
           l->l_mutex = l->l_cpu->ci_schedstate.spc_lwplock;
           l->l_ts = pool_get(&turnstile_pool, flags);
   
           if (l->l_ts == NULL) {
                   return ENOMEM;
           } else {
                   turnstile_ctor(l->l_ts);
                   return 0;
           }
   }
   
   static void
   lwp_dtor(void *arg, void *obj)
   {
           lwp_t *l = obj;
   
           /*
            * The value of l->l_cpu must still be valid at this point.
            */
           KASSERT(l->l_cpu != NULL);
   
           /*
            * We can't return turnstile0 to the pool (it didn't come from it),
            * so if it comes up just drop it quietly and move on.
            */
           if (l->l_ts != &turnstile0)
                   pool_put(&turnstile_pool, l->l_ts);
   }
   
   /*
    * Set an LWP suspended.
    *
    * Must be called with p_lock held, and the LWP locked.  Will unlock the
    * LWP before return.
    */
   int
   lwp_suspend(struct lwp *curl, struct lwp *t)
   {
           int error;
   
           KASSERT(mutex_owned(t->l_proc->p_lock));
           KASSERT(lwp_locked(t, NULL));
   
           KASSERT(curl != t || curl->l_stat == LSONPROC);
   
           /*
            * If the current LWP has been told to exit, we must not suspend anyone
            * else or deadlock could occur.  We won't return to userspace.
            */
           if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
                   lwp_unlock(t);
                   return (EDEADLK);
           }
   
           if ((t->l_flag & LW_DBGSUSPEND) != 0) {
                   lwp_unlock(t);
                   return 0;
           }
   
           error = 0;
   
           switch (t->l_stat) {
           case LSRUN:
           case LSONPROC:
                   t->l_flag |= LW_WSUSPEND;
                   lwp_need_userret(t);
                   lwp_unlock(t);
                   break;
   
           case LSSLEEP:
                   t->l_flag |= LW_WSUSPEND;
   
                   /*
                    * Kick the LWP and try to get it to the kernel boundary
                    * so that it will release any locks that it holds.
                    * setrunnable() will release the lock.
                    */
                   if ((t->l_flag & LW_SINTR) != 0)
                           setrunnable(t);
                   else
                           lwp_unlock(t);
                   break;
   
           case LSSUSPENDED:
                   lwp_unlock(t);
                   break;
   
           case LSSTOP:
                   t->l_flag |= LW_WSUSPEND;
                   setrunnable(t);
                   break;
   
           case LSIDL:
           case LSZOMB:
                   error = EINTR; /* It's what Solaris does..... */
                   lwp_unlock(t);
                   break;
           }
   
           return (error);
   }
   
   /*
    * Restart a suspended LWP.
    *
    * Must be called with p_lock held, and the LWP locked.  Will unlock the
    * LWP before return.
    */
   void
   lwp_continue(struct lwp *l)
   {
   
           KASSERT(mutex_owned(l->l_proc->p_lock));
           KASSERT(lwp_locked(l, NULL));
   
           /* If rebooting or not suspended, then just bail out. */
           if ((l->l_flag & LW_WREBOOT) != 0) {
                   lwp_unlock(l);
                   return;
           }
   
           l->l_flag &= ~LW_WSUSPEND;
   
           if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
                   lwp_unlock(l);
                   return;
           }
   
           /* setrunnable() will release the lock. */
           setrunnable(l);
   }
   
   /*
    * Restart a stopped LWP.
    *
    * Must be called with p_lock held, and the LWP NOT locked.  Will unlock the
    * LWP before return.
    */
   void
   lwp_unstop(struct lwp *l)
   {
           struct proc *p = l->l_proc;
   
           KASSERT(mutex_owned(&proc_lock));
           KASSERT(mutex_owned(p->p_lock));
   
           lwp_lock(l);
   
           KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
   
           /* If not stopped, then just bail out. */
           if (l->l_stat != LSSTOP) {
                   lwp_unlock(l);
                   return;
           }
   
           p->p_stat = SACTIVE;
           p->p_sflag &= ~PS_STOPPING;
   
           if (!p->p_waited)
                   p->p_pptr->p_nstopchild--;
   
           if (l->l_wchan == NULL) {
                   /* setrunnable() will release the lock. */
                   setrunnable(l);
           } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
                   /* setrunnable() so we can receive the signal */
                   setrunnable(l);
           } else {
                   l->l_stat = LSSLEEP;
                   p->p_nrlwps++;
                   lwp_unlock(l);
           }
   }
   
   /*
    * Wait for an LWP within the current process to exit.  If 'lid' is
    * non-zero, we are waiting for a specific LWP.
    *
    * Must be called with p->p_lock held.
    */
   int
   lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
   {
           const lwpid_t curlid = l->l_lid;
           proc_t *p = l->l_proc;
           lwp_t *l2, *next;
           int error;
   
           KASSERT(mutex_owned(p->p_lock));
   
           p->p_nlwpwait++;
           l->l_waitingfor = lid;
   
           for (;;) {
                   int nfound;
   
                   /*
                    * Avoid a race between exit1() and sigexit(): if the
                    * process is dumping core, then we need to bail out: call
                    * into lwp_userret() where we will be suspended until the
                    * deed is done.
                    */
                   if ((p->p_sflag & PS_WCORE) != 0) {
                           mutex_exit(p->p_lock);
                           lwp_userret(l);
                           KASSERT(false);
                   }
   
                   /*
                    * First off, drain any detached LWP that is waiting to be
                    * reaped.
                    */
                   while ((l2 = p->p_zomblwp) != NULL) {
                           p->p_zomblwp = NULL;
                           lwp_free(l2, false, false);/* releases proc mutex */
                           mutex_enter(p->p_lock);
                   }
   
                   /*
                    * Now look for an LWP to collect.  If the whole process is
                    * exiting, count detached LWPs as eligible to be collected,
                    * but don't drain them here.
                    */
                   nfound = 0;
                   error = 0;
   
                   /*
                    * If given a specific LID, go via pid_table and make sure
                    * it's not detached.
                    */
                   if (lid != 0) {
                           l2 = proc_find_lwp(p, lid);
                           if (l2 == NULL) {
                                   error = ESRCH;
                                   break;
                           }
                           KASSERT(l2->l_lid == lid);
                           if ((l2->l_prflag & LPR_DETACHED) != 0) {
                                   error = EINVAL;
                                   break;
                           }
                   } else {
                           l2 = LIST_FIRST(&p->p_lwps);
                   }
                   for (; l2 != NULL; l2 = next) {
                           next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
   
                           /*
                            * If a specific wait and the target is waiting on
                            * us, then avoid deadlock.  This also traps LWPs
                            * that try to wait on themselves.
                            *
                            * Note that this does not handle more complicated
                            * cycles, like: t1 -> t2 -> t3 -> t1.  The process
                            * can still be killed so it is not a major problem.
                            */
                           if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
                                   error = EDEADLK;
                                   break;
                           }
                           if (l2 == l)
                                   continue;
                           if ((l2->l_prflag & LPR_DETACHED) != 0) {
                                   nfound += exiting;
                                   continue;
                           }
                           if (lid != 0) {
                                   /*
                                    * Mark this LWP as the first waiter, if there
                                    * is no other.
                                    */
                                   if (l2->l_waiter == 0)
                                           l2->l_waiter = curlid;
                           } else if (l2->l_waiter != 0) {
                                   /*
                                    * It already has a waiter - so don't
                                    * collect it.  If the waiter doesn't
                                    * grab it we'll get another chance
                                    * later.
                                    */
                                   nfound++;
                                   continue;
                           }
                           nfound++;
   
                           /* No need to lock the LWP in order to see LSZOMB. */
                           if (l2->l_stat != LSZOMB)
                                   continue;
   
                           /*
                            * We're no longer waiting.  Reset the "first waiter"
                            * pointer on the target, in case it was us.
                            */
                           l->l_waitingfor = 0;
                           l2->l_waiter = 0;
                           p->p_nlwpwait--;
                           if (departed)
                                   *departed = l2->l_lid;
                           sched_lwp_collect(l2);
   
                           /* lwp_free() releases the proc lock. */
                           lwp_free(l2, false, false);
                           mutex_enter(p->p_lock);
                           return 0;
                   }
   
                   if (error != 0)
                           break;
                   if (nfound == 0) {
                           error = ESRCH;
                           break;
                   }
   
                   /*
                    * Note: since the lock will be dropped, need to restart on
                    * wakeup to run all LWPs again, e.g. there may be new LWPs.
                    */
                   if (exiting) {
                           KASSERT(p->p_nlwps > 1);
                           error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
                           break;
                   }
   
                   /*
                    * Break out if all LWPs are in _lwp_wait().  There are
                    * other ways to hang the process with _lwp_wait(), but the
                    * sleep is interruptable so little point checking for them.
                    */
                   if (p->p_nlwpwait == p->p_nlwps) {
                           error = EDEADLK;
                           break;
                   }
   
                   /*
                    * Sit around and wait for something to happen.  We'll be 
                    * awoken if any of the conditions examined change: if an
                    * LWP exits, is collected, or is detached.
                    */
                   if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
                           break;
           }
   
           /*
            * We didn't find any LWPs to collect, we may have received a 
            * signal, or some other condition has caused us to bail out.
            *
            * If waiting on a specific LWP, clear the waiters marker: some
            * other LWP may want it.  Then, kick all the remaining waiters
            * so that they can re-check for zombies and for deadlock.
            */
           if (lid != 0) {
                   l2 = proc_find_lwp(p, lid);
                   KASSERT(l2 == NULL || l2->l_lid == lid);
   
                   if (l2 != NULL && l2->l_waiter == curlid)
                           l2->l_waiter = 0;
           }
           p->p_nlwpwait--;
           l->l_waitingfor = 0;
           cv_broadcast(&p->p_lwpcv);
   
           return error;
   }
   
   /*
    * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    * The new LWP is created in state LSIDL and must be set running,
    * suspended, or stopped by the caller.
    */
   int
   lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
       void *stack, size_t stacksize, void (*func)(void *), void *arg,
       lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
       const stack_t *sigstk)
   {
           struct lwp *l2;
   
           KASSERT(l1 == curlwp || l1->l_proc == &proc0);
   
           /*
            * Enforce limits, excluding the first lwp and kthreads.  We must
            * use the process credentials here when adjusting the limit, as
            * they are what's tied to the accounting entity.  However for
            * authorizing the action, we'll use the LWP's credentials.
            */
           mutex_enter(p2->p_lock);
           if (p2->p_nlwps != 0 && p2 != &proc0) {
                   uid_t uid = kauth_cred_getuid(p2->p_cred);
                   int count = chglwpcnt(uid, 1);
                   if (__predict_false(count >
                       p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
                           if (kauth_authorize_process(l1->l_cred,
                               KAUTH_PROCESS_RLIMIT, p2,
                               KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
                               &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
                               != 0) {
                                   (void)chglwpcnt(uid, -1);
                                   mutex_exit(p2->p_lock);
                                   return EAGAIN;
                           }
                   }
           }
   
           /*
            * First off, reap any detached LWP waiting to be collected.
            * We can re-use its LWP structure and turnstile.
            */
           if ((l2 = p2->p_zomblwp) != NULL) {
                   p2->p_zomblwp = NULL;
                   lwp_free(l2, true, false);
                   /* p2 now unlocked by lwp_free() */
                   KASSERT(l2->l_ts != NULL);
                   KASSERT(l2->l_inheritedprio == -1);
                   KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
                   memset(&l2->l_startzero, 0, sizeof(*l2) -
                       offsetof(lwp_t, l_startzero));
           } else {
                   mutex_exit(p2->p_lock);
                   l2 = pool_cache_get(lwp_cache, PR_WAITOK);
                   memset(&l2->l_startzero, 0, sizeof(*l2) -
                       offsetof(lwp_t, l_startzero));
                   SLIST_INIT(&l2->l_pi_lenders);
           }
   
           /*
            * Because of lockless lookup via pid_table, the LWP can be locked
            * and inspected briefly even after it's freed, so a few fields are
            * kept stable.
            */
           KASSERT(l2->l_stat == LSIDL);
           KASSERT(l2->l_cpu != NULL);
           KASSERT(l2->l_ts != NULL);
           KASSERT(l2->l_mutex == l2->l_cpu->ci_schedstate.spc_lwplock);
   
           l2->l_proc = p2;
           l2->l_refcnt = 0;
           l2->l_class = sclass;
   
           /*
            * Allocate a process ID for this LWP.  We need to do this now
            * while we can still unwind if it fails.  Because we're marked
            * as LSIDL, no lookups by the ID will succeed.
            *
            * N.B. this will always succeed for the first LWP in a process,
            * because proc_alloc_lwpid() will usurp the slot.  Also note
            * that l2->l_proc MUST be valid so that lookups of the proc
            * will succeed, even if the LWP itself is not visible.
            */
           if (__predict_false(proc_alloc_lwpid(p2, l2) == -1)) {
                   pool_cache_put(lwp_cache, l2);
                   return EAGAIN;
           }
   
           /*
            * If vfork(), we want the LWP to run fast and on the same CPU
            * as its parent, so that it can reuse the VM context and cache
            * footprint on the local CPU.
            */
           l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
           l2->l_kpribase = PRI_KERNEL;
           l2->l_priority = l1->l_priority;
           l2->l_inheritedprio = -1;
           l2->l_protectprio = -1;
           l2->l_auxprio = -1;
           l2->l_flag = 0;
           l2->l_pflag = LP_MPSAFE;
           TAILQ_INIT(&l2->l_ld_locks);
           l2->l_psrefs = 0;
           kmsan_lwp_alloc(l2);
   
           /*
            * For vfork, borrow parent's lwpctl context if it exists.
            * This also causes us to return via lwp_userret.
            */
           if (flags & LWP_VFORK && l1->l_lwpctl) {
                   l2->l_lwpctl = l1->l_lwpctl;
                   l2->l_flag |= LW_LWPCTL;
           }
   
           /*
            * If not the first LWP in the process, grab a reference to the
            * descriptor table.
            */
           l2->l_fd = p2->p_fd;
           if (p2->p_nlwps != 0) {
                   KASSERT(l1->l_proc == p2);
                   fd_hold(l2);
           } else {
                   KASSERT(l1->l_proc != p2);
           }
   
           if (p2->p_flag & PK_SYSTEM) {
                   /* Mark it as a system LWP. */
                   l2->l_flag |= LW_SYSTEM;
           }
   
           kdtrace_thread_ctor(NULL, l2);
           lwp_initspecific(l2);
           sched_lwp_fork(l1, l2);
           lwp_update_creds(l2);
           callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
           callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
           cv_init(&l2->l_sigcv, "sigwait");
           cv_init(&l2->l_waitcv, "vfork");
           l2->l_syncobj = &sched_syncobj;
           PSREF_DEBUG_INIT_LWP(l2);
   
           if (rnewlwpp != NULL)
                   *rnewlwpp = l2;
   
           /*
            * PCU state needs to be saved before calling uvm_lwp_fork() so that
            * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
            */
           pcu_save_all(l1);
   #if PCU_UNIT_COUNT > 0
           l2->l_pcu_valid = l1->l_pcu_valid;
   #endif
   
           uvm_lwp_setuarea(l2, uaddr);
           uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
   
           mutex_enter(p2->p_lock);
           if ((flags & LWP_DETACHED) != 0) {
                   l2->l_prflag = LPR_DETACHED;
                   p2->p_ndlwps++;
           } else
                   l2->l_prflag = 0;
   
           if (l1->l_proc == p2) {
                   /*
                    * These flags are set while p_lock is held.  Copy with
                    * p_lock held too, so the LWP doesn't sneak into the
                    * process without them being set.
                    */
                   l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
           } else {
                   /* fork(): pending core/exit doesn't apply to child. */
                   l2->l_flag |= (l1->l_flag & LW_WREBOOT);
           }
   
           l2->l_sigstk = *sigstk;
           l2->l_sigmask = *sigmask;
           TAILQ_INIT(&l2->l_sigpend.sp_info);
           sigemptyset(&l2->l_sigpend.sp_set);
           LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
           p2->p_nlwps++;
           p2->p_nrlwps++;
   
           KASSERT(l2->l_affinity == NULL);
   
           /* Inherit the affinity mask. */
           if (l1->l_affinity) {
                   /*
                    * Note that we hold the state lock while inheriting
                    * the affinity to avoid race with sched_setaffinity().
                    */
                   lwp_lock(l1);
                   if (l1->l_affinity) {
                           kcpuset_use(l1->l_affinity);
                           l2->l_affinity = l1->l_affinity;
                   }
                   lwp_unlock(l1);
           }
   
           /* This marks the end of the "must be atomic" section. */
           mutex_exit(p2->p_lock);
   
           SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
   
           mutex_enter(&proc_lock);
           LIST_INSERT_HEAD(&alllwp, l2, l_list);
           /* Inherit a processor-set */
           l2->l_psid = l1->l_psid;
           mutex_exit(&proc_lock);
   
           SYSCALL_TIME_LWP_INIT(l2);
   
           if (p2->p_emul->e_lwp_fork)
                   (*p2->p_emul->e_lwp_fork)(l1, l2);
   
           return (0);
   }
   
   /*
    * Set a new LWP running.  If the process is stopping, then the LWP is
    * created stopped.
    */
   void
   lwp_start(lwp_t *l, int flags)
   {
           proc_t *p = l->l_proc;
   
           mutex_enter(p->p_lock);
           lwp_lock(l);
           KASSERT(l->l_stat == LSIDL);
           if ((flags & LWP_SUSPENDED) != 0) {
                   /* It'll suspend itself in lwp_userret(). */
                   l->l_flag |= LW_WSUSPEND;
           }
          if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
                  KASSERT(l->l_wchan == NULL);
                  l->l_stat = LSSTOP;
                  p->p_nrlwps--;
                  lwp_unlock(l);
          } else {
                  setrunnable(l);
                  /* LWP now unlocked */
          }
          mutex_exit(p->p_lock);
  }
  
  /*
   * Called by MD code when a new LWP begins execution.  Must be called
   * with the previous LWP locked (so at splsched), or if there is no
   * previous LWP, at splsched.
   */
  void
  lwp_startup(struct lwp *prev, struct lwp *new_lwp)
  {
          kmutex_t *lock;
  
          KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
          KASSERT(kpreempt_disabled());
          KASSERT(prev != NULL);
          KASSERT((prev->l_pflag & LP_RUNNING) != 0);
          KASSERT(curcpu()->ci_mtx_count == -2);
  
          /*
           * Immediately mark the previous LWP as no longer running and
           * unlock (to keep lock wait times short as possible).  If a
           * zombie, don't touch after clearing LP_RUNNING as it could be
           * reaped by another CPU.  Use atomic_store_release to ensure
           * this -- matches atomic_load_acquire in lwp_free.
           */
          lock = prev->l_mutex;
          if (__predict_false(prev->l_stat == LSZOMB)) {
                  atomic_store_release(&prev->l_pflag,
                      prev->l_pflag & ~LP_RUNNING);
          } else {
                  prev->l_pflag &= ~LP_RUNNING;
          }
          mutex_spin_exit(lock);
  
          /* Correct spin mutex count after mi_switch(). */
          curcpu()->ci_mtx_count = 0;
  
          /* Install new VM context. */
          if (__predict_true(new_lwp->l_proc->p_vmspace)) {
                  pmap_activate(new_lwp);
          }
  
          /* We remain at IPL_SCHED from mi_switch() - reset it. */
          spl0();
  
          LOCKDEBUG_BARRIER(NULL, 0);
          SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
  
          /* For kthreads, acquire kernel lock if not MPSAFE. */
          if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
                  KERNEL_LOCK(1, new_lwp);
          }
  }
  
  /*
   * Exit an LWP.
   *
   * *** WARNING *** This can be called with (l != curlwp) in error paths.
   */
  void
  lwp_exit(struct lwp *l)
  {
          struct proc *p = l->l_proc;
          struct lwp *l2;
          bool current;
  
          current = (l == curlwp);
  
          KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
          KASSERT(p == curproc);
  
          SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
  
          /* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
          LOCKDEBUG_BARRIER(NULL, 0);
          KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
  
          /*
           * If we are the last live LWP in a process, we need to exit the
           * entire process.  We do so with an exit status of zero, because
           * it's a "controlled" exit, and because that's what Solaris does.
           *
           * We are not quite a zombie yet, but for accounting purposes we
           * must increment the count of zombies here.
           *
           * Note: the last LWP's specificdata will be deleted here.
           */
          mutex_enter(p->p_lock);
          if (p->p_nlwps - p->p_nzlwps == 1) {
                  KASSERT(current == true);
                  KASSERT(p != &proc0);
                  exit1(l, 0, 0);
                  /* NOTREACHED */
          }
          p->p_nzlwps++;
  
          /*
           * Perform any required thread cleanup.  Do this early so
           * anyone wanting to look us up with lwp_getref_lwpid() will
           * fail to find us before we become a zombie.
           *
           * N.B. this will unlock p->p_lock on our behalf.
           */
          lwp_thread_cleanup(l);
  
          if (p->p_emul->e_lwp_exit)
                  (*p->p_emul->e_lwp_exit)(l);
  
          /* Drop filedesc reference. */
          fd_free();
  
          /* Release fstrans private data. */
          fstrans_lwp_dtor(l);
  
          /* Delete the specificdata while it's still safe to sleep. */
          lwp_finispecific(l);
  
          /*
           * Release our cached credentials.
           */
          kauth_cred_free(l->l_cred);
          callout_destroy(&l->l_timeout_ch);
  
          /*
           * If traced, report LWP exit event to the debugger.
           *
           * Remove the LWP from the global list.
           * Free its LID from the PID namespace if needed.
           */
          mutex_enter(&proc_lock);
  
          if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
              (PSL_TRACED|PSL_TRACELWP_EXIT)) {
                  mutex_enter(p->p_lock);
                  if (ISSET(p->p_sflag, PS_WEXIT)) {
                          mutex_exit(p->p_lock);
                          /*
                           * We are exiting, bail out without informing parent
                           * about a terminating LWP as it would deadlock.
                           */
                  } else {
                          eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
                          mutex_enter(&proc_lock);
                  }
          }
  
          LIST_REMOVE(l, l_list);
          mutex_exit(&proc_lock);
  
          /*
           * Get rid of all references to the LWP that others (e.g. procfs)
           * may have, and mark the LWP as a zombie.  If the LWP is detached,
           * mark it waiting for collection in the proc structure.  Note that
           * before we can do that, we need to free any other dead, deatched
           * LWP waiting to meet its maker.
           *
           * All conditions need to be observed upon under the same hold of
           * p_lock, because if the lock is dropped any of them can change.
           */
          mutex_enter(p->p_lock);
          for (;;) {
                  if (lwp_drainrefs(l))
                          continue;
                  if ((l->l_prflag & LPR_DETACHED) != 0) {
                          if ((l2 = p->p_zomblwp) != NULL) {
                                  p->p_zomblwp = NULL;
                                  lwp_free(l2, false, false);
                                  /* proc now unlocked */
                                  mutex_enter(p->p_lock);
                                  continue;
                          }
                          p->p_zomblwp = l;
                  }
                  break;
          }
  
          /*
           * If we find a pending signal for the process and we have been
           * asked to check for signals, then we lose: arrange to have
           * all other LWPs in the process check for signals.
           */
          if ((l->l_flag & LW_PENDSIG) != 0 &&
              firstsig(&p->p_sigpend.sp_set) != 0) {
                  LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
                          lwp_lock(l2);
                          signotify(l2);
                          lwp_unlock(l2);
                  }
          }
  
          /*
           * Release any PCU resources before becoming a zombie.
           */
          pcu_discard_all(l);
  
          lwp_lock(l);
          l->l_stat = LSZOMB;
          if (l->l_name != NULL) {
                  strcpy(l->l_name, "(zombie)");
          }
          lwp_unlock(l);
          p->p_nrlwps--;
          cv_broadcast(&p->p_lwpcv);
          if (l->l_lwpctl != NULL)
                  l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
          mutex_exit(p->p_lock);
  
          /*
           * We can no longer block.  At this point, lwp_free() may already
           * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
           *
           * Free MD LWP resources.
           */
          cpu_lwp_free(l, 0);
  
          if (current) {
                  /* Switch away into oblivion. */
                  lwp_lock(l);
                  spc_lock(l->l_cpu);
                  mi_switch(l);
                  panic("lwp_exit");
          }
  }
  
  /*
   * Free a dead LWP's remaining resources.
   *
   * XXXLWP limits.
   */
  void
  lwp_free(struct lwp *l, bool recycle, bool last)
  {
          struct proc *p = l->l_proc;
          struct rusage *ru;
          ksiginfoq_t kq;
  
          KASSERT(l != curlwp);
          KASSERT(last || mutex_owned(p->p_lock));
  
          /*
           * We use the process credentials instead of the lwp credentials here
           * because the lwp credentials maybe cached (just after a setuid call)
           * and we don't want pay for syncing, since the lwp is going away
           * anyway
           */
          if (p != &proc0 && p->p_nlwps != 1)
                  (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
  
          /*
           * In the unlikely event that the LWP is still on the CPU,
           * then spin until it has switched away.
           *
           * atomic_load_acquire matches atomic_store_release in
           * lwp_startup and mi_switch.
           */
          while (__predict_false((atomic_load_acquire(&l->l_pflag) & LP_RUNNING)
                  != 0)) {
                  SPINLOCK_BACKOFF_HOOK;
          }
  
          /*
           * Now that the LWP's known off the CPU, reset its state back to
           * LSIDL, which defeats anything that might have gotten a hold on
           * the LWP via pid_table before the ID was freed.  It's important
           * to do this with both the LWP locked and p_lock held.
           *
           * Also reset the CPU and lock pointer back to curcpu(), since the
           * LWP will in all likelyhood be cached with the current CPU in
           * lwp_cache when we free it and later allocated from there again
           * (avoid incidental lock contention).
           */
          lwp_lock(l);
          l->l_stat = LSIDL;
          l->l_cpu = curcpu();
          lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_lwplock);
  
          /*
           * If this was not the last LWP in the process, then adjust counters
           * and unlock.  This is done differently for the last LWP in exit1().
           */
          if (!last) {
                  /*
                   * Add the LWP's run time to the process' base value.
                   * This needs to co-incide with coming off p_lwps.
                   */
                  bintime_add(&p->p_rtime, &l->l_rtime);
                  p->p_pctcpu += l->l_pctcpu;
                  ru = &p->p_stats->p_ru;
                  ruadd(ru, &l->l_ru);
                  ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
                  ru->ru_nivcsw += l->l_nivcsw;
                  LIST_REMOVE(l, l_sibling);
                  p->p_nlwps--;
                  p->p_nzlwps--;
                  if ((l->l_prflag & LPR_DETACHED) != 0)
                          p->p_ndlwps--;
  
                  /*
                   * Have any LWPs sleeping in lwp_wait() recheck for
                   * deadlock.
                   */
                  cv_broadcast(&p->p_lwpcv);
                  mutex_exit(p->p_lock);
  
                  /* Free the LWP ID. */
                  mutex_enter(&proc_lock);
                  proc_free_lwpid(p, l->l_lid);
                  mutex_exit(&proc_lock);
          }
  
          /*
           * Destroy the LWP's remaining signal information.
           */
          ksiginfo_queue_init(&kq);
          sigclear(&l->l_sigpend, NULL, &kq);
          ksiginfo_queue_drain(&kq);
          cv_destroy(&l->l_sigcv);
          cv_destroy(&l->l_waitcv);
  
          /*
           * Free lwpctl structure and affinity.
           */
          if (l->l_lwpctl) {
                  lwp_ctl_free(l);
          }
          if (l->l_affinity) {
                  kcpuset_unuse(l->l_affinity, NULL);
                  l->l_affinity = NULL;
          }
  
          /*
           * Free remaining data structures and the LWP itself unless the
           * caller wants to recycle.
           */
          if (l->l_name != NULL)
                  kmem_free(l->l_name, MAXCOMLEN);
  
          kmsan_lwp_free(l);
          kcov_lwp_free(l);
          cpu_lwp_free2(l);
          uvm_lwp_exit(l);
  
          KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
          KASSERT(l->l_inheritedprio == -1);
          KASSERT(l->l_blcnt == 0);
          kdtrace_thread_dtor(NULL, l);
          if (!recycle)
                  pool_cache_put(lwp_cache, l);
  }
  
  /*
   * Migrate the LWP to the another CPU.  Unlocks the LWP.
   */
  void
  lwp_migrate(lwp_t *l, struct cpu_info *tci)
  {
          struct schedstate_percpu *tspc;
          int lstat = l->l_stat;
  
          KASSERT(lwp_locked(l, NULL));
          KASSERT(tci != NULL);
  
          /* If LWP is still on the CPU, it must be handled like LSONPROC */
          if ((l->l_pflag & LP_RUNNING) != 0) {
                  lstat = LSONPROC;
          }
  
          /*
           * The destination CPU could be changed while previous migration
           * was not finished.
           */
          if (l->l_target_cpu != NULL) {
                  l->l_target_cpu = tci;
                  lwp_unlock(l);
                  return;
          }
  
          /* Nothing to do if trying to migrate to the same CPU */
          if (l->l_cpu == tci) {
                  lwp_unlock(l);
                  return;
          }
  
          KASSERT(l->l_target_cpu == NULL);
          tspc = &tci->ci_schedstate;
          switch (lstat) {
          case LSRUN:
                  l->l_target_cpu = tci;
                  break;
          case LSSLEEP:
                  l->l_cpu = tci;
                  break;
          case LSIDL:
          case LSSTOP:
          case LSSUSPENDED:
                  l->l_cpu = tci;
                  if (l->l_wchan == NULL) {
                          lwp_unlock_to(l, tspc->spc_lwplock);
                          return;
                  }
                  break;
          case LSONPROC:
                  l->l_target_cpu = tci;
                  spc_lock(l->l_cpu);
                  sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
                  /* spc now unlocked */
                  break;
          }
          lwp_unlock(l);
  }
  
  #define lwp_find_exclude(l)                                     \
          ((l)->l_stat == LSIDL || (l)->l_stat == LSZOMB)
  
  /*
   * Find the LWP in the process.  Arguments may be zero, in such case,
   * the calling process and first LWP in the list will be used.
   * On success - returns proc locked.
   *
   * => pid == 0 -> look in curproc.
   * => pid == -1 -> match any proc.
   * => otherwise look up the proc.
   *
   * => lid == 0 -> first LWP in the proc
   * => otherwise specific LWP
   */
  struct lwp *
  lwp_find2(pid_t pid, lwpid_t lid)
  {
          proc_t *p;
          lwp_t *l;
  
          /* First LWP of specified proc. */
          if (lid == 0) {
                  switch (pid) {
                  case -1:
                          /* No lookup keys. */
                          return NULL;
                  case 0:
                          p = curproc;
                          mutex_enter(p->p_lock);
                          break;
                  default:
                          mutex_enter(&proc_lock);
                          p = proc_find(pid);
                          if (__predict_false(p == NULL)) {
                                  mutex_exit(&proc_lock);
                                  return NULL;
                          }
                          mutex_enter(p->p_lock);
                          mutex_exit(&proc_lock);
                          break;
                  }
                  LIST_FOREACH(l, &p->p_lwps, l_sibling) {
                          if (__predict_true(!lwp_find_exclude(l)))
                                  break;
                  }
                  goto out;
          }
  
          l = proc_find_lwp_acquire_proc(lid, &p);
          if (l == NULL)
                  return NULL;
          KASSERT(p != NULL);
          KASSERT(mutex_owned(p->p_lock));
  
          if (__predict_false(lwp_find_exclude(l))) {
                  l = NULL;
                  goto out;
          }
  
          /* Apply proc filter, if applicable. */
          switch (pid) {
          case -1:
                  /* Match anything. */
                  break;
          case 0:
                  if (p != curproc)
                          l = NULL;
                  break;
          default:
                  if (p->p_pid != pid)
                          l = NULL;
                  break;
          }
  
   out:
          if (__predict_false(l == NULL)) {
                  mutex_exit(p->p_lock);
          }
          return l;
  }
  
  /*
   * Look up a live LWP within the specified process.
   *
   * Must be called with p->p_lock held (as it looks at the radix tree,
   * and also wants to exclude idle and zombie LWPs).
   */
  struct lwp *
  lwp_find(struct proc *p, lwpid_t id)
  {
          struct lwp *l;
  
          KASSERT(mutex_owned(p->p_lock));
  
          l = proc_find_lwp(p, id);
          KASSERT(l == NULL || l->l_lid == id);
  
          /*
           * No need to lock - all of these conditions will
           * be visible with the process level mutex held.
           */
          if (__predict_false(l != NULL && lwp_find_exclude(l)))
                  l = NULL;
  
          return l;
  }
  
  /*
   * Update an LWP's cached credentials to mirror the process' master copy.
   *
   * This happens early in the syscall path, on user trap, and on LWP
   * creation.  A long-running LWP can also voluntarily choose to update
   * its credentials by calling this routine.  This may be called from
   * LWP_CACHE_CREDS(), which checks l->l_prflag & LPR_CRMOD beforehand.
   */
  void
  lwp_update_creds(struct lwp *l)
  {
          kauth_cred_t oc;
          struct proc *p;
  
          p = l->l_proc;
          oc = l->l_cred;
  
          mutex_enter(p->p_lock);
          kauth_cred_hold(p->p_cred);
          l->l_cred = p->p_cred;
          l->l_prflag &= ~LPR_CRMOD;
          mutex_exit(p->p_lock);
          if (oc != NULL)
                  kauth_cred_free(oc);
  }
  
  /*
   * Verify that an LWP is locked, and optionally verify that the lock matches
   * one we specify.
   */
  int
  lwp_locked(struct lwp *l, kmutex_t *mtx)
  {
          kmutex_t *cur = l->l_mutex;
  
          return mutex_owned(cur) && (mtx == cur || mtx == NULL);
  }
  
  /*
   * Lend a new mutex to an LWP.  The old mutex must be held.
   */
  kmutex_t *
  lwp_setlock(struct lwp *l, kmutex_t *mtx)
  {
          kmutex_t *oldmtx = l->l_mutex;
  
          KASSERT(mutex_owned(oldmtx));
  
          atomic_store_release(&l->l_mutex, mtx);
          return oldmtx;
  }
  
  /*
   * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   * must be held.
   */
  void
  lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
  {
          kmutex_t *old;
  
          KASSERT(lwp_locked(l, NULL));
  
          old = l->l_mutex;
          atomic_store_release(&l->l_mutex, mtx);
          mutex_spin_exit(old);
  }
  
  int
  lwp_trylock(struct lwp *l)
  {
          kmutex_t *old;
  
          for (;;) {
                  if (!mutex_tryenter(old = atomic_load_consume(&l->l_mutex)))
                          return 0;
                  if (__predict_true(atomic_load_relaxed(&l->l_mutex) == old))
                          return 1;
                  mutex_spin_exit(old);
          }
  }
  
  void
  lwp_unsleep(lwp_t *l, bool unlock)
  {
  
          KASSERT(mutex_owned(l->l_mutex));
          (*l->l_syncobj->sobj_unsleep)(l, unlock);
  }
  
  /*
   * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   * set.
   */
  void
  lwp_userret(struct lwp *l)
  {
          struct proc *p;
          int sig;
  
          KASSERT(l == curlwp);
          KASSERT(l->l_stat == LSONPROC);
          p = l->l_proc;
  
          /*
           * It is safe to do this read unlocked on a MP system..
           */
          while ((l->l_flag & LW_USERRET) != 0) {
                  /*
                   * Process pending signals first, unless the process
                   * is dumping core or exiting, where we will instead
                   * enter the LW_WSUSPEND case below.
                   */
                  if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
                      LW_PENDSIG) {
                          mutex_enter(p->p_lock);
                          while ((sig = issignal(l)) != 0)
                                  postsig(sig);
                          mutex_exit(p->p_lock);
                  }
  
                  /*
                   * Core-dump or suspend pending.
                   *
                   * In case of core dump, suspend ourselves, so that the kernel
                   * stack and therefore the userland registers saved in the
                   * trapframe are around for coredump() to write them out.
                   * We also need to save any PCU resources that we have so that
                   * they accessible for coredump().  We issue a wakeup on
                   * p->p_lwpcv so that sigexit() will write the core file out
                   * once all other LWPs are suspended.  
                   */
                  if ((l->l_flag & LW_WSUSPEND) != 0) {
                          pcu_save_all(l);
                          mutex_enter(p->p_lock);
                          p->p_nrlwps--;
                          cv_broadcast(&p->p_lwpcv);
                          lwp_lock(l);
                          l->l_stat = LSSUSPENDED;
                          lwp_unlock(l);
                          mutex_exit(p->p_lock);
                          lwp_lock(l);
                          spc_lock(l->l_cpu);
                          mi_switch(l);
                  }
  
                  /* Process is exiting. */
                  if ((l->l_flag & LW_WEXIT) != 0) {
                          lwp_exit(l);
                          KASSERT(0);
                          /* NOTREACHED */
                  }
  
                  /* update lwpctl processor (for vfork child_return) */
                  if (l->l_flag & LW_LWPCTL) {
                          lwp_lock(l);
                          KASSERT(kpreempt_disabled());
                          l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
                          l->l_lwpctl->lc_pctr++;
                          l->l_flag &= ~LW_LWPCTL;
                          lwp_unlock(l);
                  }
          }
  }
  
  /*
   * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   */
  void
  lwp_need_userret(struct lwp *l)
  {
  
          KASSERT(!cpu_intr_p());
          KASSERT(lwp_locked(l, NULL));
  
          /*
           * If the LWP is in any state other than LSONPROC, we know that it
           * is executing in-kernel and will hit userret() on the way out. 
           *
           * If the LWP is curlwp, then we know we'll be back out to userspace
           * soon (can't be called from a hardware interrupt here).
           *
           * Otherwise, we can't be sure what the LWP is doing, so first make
           * sure the update to l_flag will be globally visible, and then
           * force the LWP to take a trip through trap() where it will do
           * userret().
           */
          if (l->l_stat == LSONPROC && l != curlwp) {
                  membar_producer();
                  cpu_signotify(l);
          }
  }
  
  /*
   * Add one reference to an LWP.  This will prevent the LWP from
   * exiting, thus keep the lwp structure and PCB around to inspect.
   */
  void
  lwp_addref(struct lwp *l)
  {
          KASSERT(mutex_owned(l->l_proc->p_lock));
          KASSERT(l->l_stat != LSZOMB);
          l->l_refcnt++;
  }
  
  /*
   * Remove one reference to an LWP.  If this is the last reference,
   * then we must finalize the LWP's death.
   */
  void
  lwp_delref(struct lwp *l)
  {
          struct proc *p = l->l_proc;
  
          mutex_enter(p->p_lock);
          lwp_delref2(l);
          mutex_exit(p->p_lock);
  }
  
  /*
   * Remove one reference to an LWP.  If this is the last reference,
   * then we must finalize the LWP's death.  The proc mutex is held
   * on entry.
   */
  void
  lwp_delref2(struct lwp *l)
  {
          struct proc *p = l->l_proc;
  
          KASSERT(mutex_owned(p->p_lock));
          KASSERT(l->l_stat != LSZOMB);
          KASSERT(l->l_refcnt > 0);
  
          if (--l->l_refcnt == 0)
                  cv_broadcast(&p->p_lwpcv);
  }
  
  /*
   * Drain all references to the current LWP.  Returns true if
   * we blocked.
   */
  bool
  lwp_drainrefs(struct lwp *l)
  {
          struct proc *p = l->l_proc;
          bool rv = false;
  
          KASSERT(mutex_owned(p->p_lock));
  
          l->l_prflag |= LPR_DRAINING;
  
          while (l->l_refcnt > 0) {
                  rv = true;
                  cv_wait(&p->p_lwpcv, p->p_lock);
          }
          return rv;
  }
  
  /*
   * Return true if the specified LWP is 'alive'.  Only p->p_lock need
   * be held.
   */
  bool
  lwp_alive(lwp_t *l)
  {
  
          KASSERT(mutex_owned(l->l_proc->p_lock));
  
          switch (l->l_stat) {
          case LSSLEEP:
          case LSRUN:
          case LSONPROC:
          case LSSTOP:
          case LSSUSPENDED:
                  return true;
          default:
                  return false;
          }
  }
  
  /*
   * Return first live LWP in the process.
   */
  lwp_t *
  lwp_find_first(proc_t *p)
  {
          lwp_t *l;
  
          KASSERT(mutex_owned(p->p_lock));
  
          LIST_FOREACH(l, &p->p_lwps, l_sibling) {
                  if (lwp_alive(l)) {
                          return l;
                  }
          }
  
          return NULL;
  }
  
  /*
   * Allocate a new lwpctl structure for a user LWP.
   */
  int
  lwp_ctl_alloc(vaddr_t *uaddr)
  {
          lcproc_t *lp;
          u_int bit, i, offset;
          struct uvm_object *uao;
          int error;
          lcpage_t *lcp;
          proc_t *p;
          lwp_t *l;
  
          l = curlwp;
          p = l->l_proc;
  
          /* don't allow a vforked process to create lwp ctls */
          if (p->p_lflag & PL_PPWAIT)
                  return EBUSY;
  
          if (l->l_lcpage != NULL) {
                  lcp = l->l_lcpage;
                  *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
                  return 0;
          }
  
          /* First time around, allocate header structure for the process. */
          if ((lp = p->p_lwpctl) == NULL) {
                  lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
                  mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
                  lp->lp_uao = NULL;
                  TAILQ_INIT(&lp->lp_pages);
                  mutex_enter(p->p_lock);
                  if (p->p_lwpctl == NULL) {
                          p->p_lwpctl = lp;
                          mutex_exit(p->p_lock);
                  } else {
                          mutex_exit(p->p_lock);
                          mutex_destroy(&lp->lp_lock);
                          kmem_free(lp, sizeof(*lp));
                          lp = p->p_lwpctl;
                  }
          }
  
          /*
           * Set up an anonymous memory region to hold the shared pages.
           * Map them into the process' address space.  The user vmspace
           * gets the first reference on the UAO.
           */
          mutex_enter(&lp->lp_lock);
          if (lp->lp_uao == NULL) {
                  lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
                  lp->lp_cur = 0;
                  lp->lp_max = LWPCTL_UAREA_SZ;
                  lp->lp_uva = p->p_emul->e_vm_default_addr(p,
                       (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
                       p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
                  error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
                      LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
                      UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
                  if (error != 0) {
                          uao_detach(lp->lp_uao);
                          lp->lp_uao = NULL;
                          mutex_exit(&lp->lp_lock);
                          return error;
                  }
          }
  
          /* Get a free block and allocate for this LWP. */
          TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
                  if (lcp->lcp_nfree != 0)
                          break;
          }
          if (lcp == NULL) {
                  /* Nothing available - try to set up a free page. */
                  if (lp->lp_cur == lp->lp_max) {
                          mutex_exit(&lp->lp_lock);
                          return ENOMEM;
                  }
                  lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
  
                  /*
                   * Wire the next page down in kernel space.  Since this
                   * is a new mapping, we must add a reference.
                   */
                  uao = lp->lp_uao;
                  (*uao->pgops->pgo_reference)(uao);
                  lcp->lcp_kaddr = vm_map_min(kernel_map);
                  error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
                      uao, lp->lp_cur, PAGE_SIZE,
                      UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
                      UVM_INH_NONE, UVM_ADV_RANDOM, 0));
                  if (error != 0) {
                          mutex_exit(&lp->lp_lock);
                          kmem_free(lcp, LWPCTL_LCPAGE_SZ);
                          (*uao->pgops->pgo_detach)(uao);
                          return error;
                  }
                  error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
                      lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
                  if (error != 0) {
                          mutex_exit(&lp->lp_lock);
                          uvm_unmap(kernel_map, lcp->lcp_kaddr,
                              lcp->lcp_kaddr + PAGE_SIZE);
                          kmem_free(lcp, LWPCTL_LCPAGE_SZ);
                          return error;
                  }
                  /* Prepare the page descriptor and link into the list. */
                  lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
                  lp->lp_cur += PAGE_SIZE;
                  lcp->lcp_nfree = LWPCTL_PER_PAGE;
                  lcp->lcp_rotor = 0;
                  memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
                  TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
          }
          for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
                  if (++i >= LWPCTL_BITMAP_ENTRIES)
                          i = 0;
          }
          bit = ffs(lcp->lcp_bitmap[i]) - 1;
          lcp->lcp_bitmap[i] ^= (1U << bit);
          lcp->lcp_rotor = i;
          lcp->lcp_nfree--;
          l->l_lcpage = lcp;
          offset = (i << 5) + bit;
          l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
          *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
          mutex_exit(&lp->lp_lock);
  
          KPREEMPT_DISABLE(l);
          l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
          KPREEMPT_ENABLE(l);
  
          return 0;
  }
  
  /*
   * Free an lwpctl structure back to the per-process list.
   */
  void
  lwp_ctl_free(lwp_t *l)
  {
          struct proc *p = l->l_proc;
          lcproc_t *lp;
          lcpage_t *lcp;
          u_int map, offset;
  
          /* don't free a lwp context we borrowed for vfork */
          if (p->p_lflag & PL_PPWAIT) {
                  l->l_lwpctl = NULL;
                  return;
          }
  
          lp = p->p_lwpctl;
          KASSERT(lp != NULL);
  
          lcp = l->l_lcpage;
          offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
          KASSERT(offset < LWPCTL_PER_PAGE);
  
          mutex_enter(&lp->lp_lock);
          lcp->lcp_nfree++;
          map = offset >> 5;
          lcp->lcp_bitmap[map] |= (1U << (offset & 31));
          if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
                  lcp->lcp_rotor = map;
          if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
                  TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
                  TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
          }
          mutex_exit(&lp->lp_lock);
  }
  
  /*
   * Process is exiting; tear down lwpctl state.  This can only be safely
   * called by the last LWP in the process.
   */
  void
  lwp_ctl_exit(void)
  {
          lcpage_t *lcp, *next;
          lcproc_t *lp;
          proc_t *p;
          lwp_t *l;
  
          l = curlwp;
          l->l_lwpctl = NULL;
          l->l_lcpage = NULL;
          p = l->l_proc;
          lp = p->p_lwpctl;
  
          KASSERT(lp != NULL);
          KASSERT(p->p_nlwps == 1);
  
          for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
                  next = TAILQ_NEXT(lcp, lcp_chain);
                  uvm_unmap(kernel_map, lcp->lcp_kaddr,
                      lcp->lcp_kaddr + PAGE_SIZE);
                  kmem_free(lcp, LWPCTL_LCPAGE_SZ);
          }
  
          if (lp->lp_uao != NULL) {
                  uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
                      lp->lp_uva + LWPCTL_UAREA_SZ);
          }
  
          mutex_destroy(&lp->lp_lock);
          kmem_free(lp, sizeof(*lp));
          p->p_lwpctl = NULL;
  }
  
  /*
   * Return the current LWP's "preemption counter".  Used to detect
   * preemption across operations that can tolerate preemption without
   * crashing, but which may generate incorrect results if preempted.
   */
  uint64_t
  lwp_pctr(void)
  {
  
          return curlwp->l_ncsw;
  }
  
  /*
   * Set an LWP's private data pointer.
   */
  int
  lwp_setprivate(struct lwp *l, void *ptr)
  {
          int error = 0;
  
          l->l_private = ptr;
  #ifdef __HAVE_CPU_LWP_SETPRIVATE
          error = cpu_lwp_setprivate(l, ptr);
  #endif
          return error;
  }
  
  /*
   * Perform any thread-related cleanup on LWP exit.
   * N.B. l->l_proc->p_lock must be HELD on entry but will
   * be released before returning!
   */
  void
  lwp_thread_cleanup(struct lwp *l)
  {
  
          KASSERT(mutex_owned(l->l_proc->p_lock));
          mutex_exit(l->l_proc->p_lock);
  
          /*
           * If the LWP has robust futexes, release them all
           * now.
           */
          if (__predict_false(l->l_robust_head != 0)) {
                  futex_release_all_lwp(l);
          }
  }
  
  #if defined(DDB)
  #include <machine/pcb.h>
  
  void
  lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
  {
          lwp_t *l;
  
          LIST_FOREACH(l, &alllwp, l_list) {
                  uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
  
                  if (addr < stack || stack + KSTACK_SIZE <= addr) {
                          continue;
                  }
                  (*pr)("%p is %p+%zu, LWP %p's stack\n",
                      (void *)addr, (void *)stack,
                      (size_t)(addr - stack), l);
          }
  }
  #endif /* defined(DDB) */

Cache object: d1ffc46ef1cf22424f45f6472aceadaf