The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_sa.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: kern_sa.c,v 1.87.2.2 2008/09/16 18:49:34 bouyer Exp $  */
    2 
    3 /*-
    4  * Copyright (c) 2001, 2004, 2005 The NetBSD Foundation, Inc.
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to The NetBSD Foundation
    8  * by Nathan J. Williams.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 3. All advertising materials mentioning features or use of this software
   19  *    must display the following acknowledgement:
   20  *        This product includes software developed by the NetBSD
   21  *        Foundation, Inc. and its contributors.
   22  * 4. Neither the name of The NetBSD Foundation nor the names of its
   23  *    contributors may be used to endorse or promote products derived
   24  *    from this software without specific prior written permission.
   25  *
   26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   36  * POSSIBILITY OF SUCH DAMAGE.
   37  */
   38 
   39 #include <sys/cdefs.h>
   40 
   41 #include "opt_ktrace.h"
   42 #include "opt_multiprocessor.h"
   43 __KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.87.2.2 2008/09/16 18:49:34 bouyer Exp $");
   44 
   45 #include <sys/param.h>
   46 #include <sys/systm.h>
   47 #include <sys/pool.h>
   48 #include <sys/proc.h>
   49 #include <sys/types.h>
   50 #include <sys/ucontext.h>
   51 #include <sys/kmem.h>
   52 #include <sys/mount.h>
   53 #include <sys/sa.h>
   54 #include <sys/savar.h>
   55 #include <sys/syscallargs.h>
   56 #include <sys/ktrace.h>
   57 
   58 #include <uvm/uvm_extern.h>
   59 
   60 static POOL_INIT(sadata_pool, sizeof(struct sadata), 0, 0, 0, "sadatapl",
   61     &pool_allocator_nointr); /* memory pool for sadata structures */
   62 static POOL_INIT(saupcall_pool, sizeof(struct sadata_upcall), 0, 0, 0,
   63     "saupcpl", &pool_allocator_nointr); /* memory pool for pending upcalls */
   64 static POOL_INIT(sastack_pool, sizeof(struct sastack), 0, 0, 0, "sastackpl",
   65     &pool_allocator_nointr); /* memory pool for sastack structs */
   66 static POOL_INIT(savp_pool, sizeof(struct sadata_vp), 0, 0, 0, "savppl",
   67     &pool_allocator_nointr); /* memory pool for sadata_vp structures */
   68 
   69 static struct sadata_vp *sa_newsavp(struct sadata *);
   70 static inline int sa_stackused(struct sastack *, struct sadata *);
   71 static inline void sa_setstackfree(struct sastack *, struct sadata *);
   72 static struct sastack *sa_getstack(struct sadata *);
   73 static inline struct sastack *sa_getstack0(struct sadata *);
   74 static inline int sast_compare(struct sastack *, struct sastack *);
   75 #ifdef MULTIPROCESSOR
   76 static int sa_increaseconcurrency(struct lwp *, int);
   77 #endif
   78 static void sa_setwoken(struct lwp *);
   79 static void sa_switchcall(void *);
   80 static int sa_newcachelwp(struct lwp *);
   81 static inline void sa_makeupcalls(struct lwp *);
   82 static struct lwp *sa_vp_repossess(struct lwp *l);
   83 
   84 static inline int sa_pagefault(struct lwp *, ucontext_t *);
   85 
   86 static void sa_upcall0(struct sadata_upcall *, int, struct lwp *, struct lwp *,
   87     size_t, void *, void (*)(void *));
   88 static void sa_upcall_getstate(union sau_state *, struct lwp *);
   89 
   90 #define SA_DEBUG
   91 
   92 #ifdef SA_DEBUG
   93 #define DPRINTF(x)      do { if (sadebug) printf_nolog x; } while (0)
   94 #define DPRINTFN(n,x)   do { if (sadebug & (1<<(n-1))) printf_nolog x; } while (0)
   95 int     sadebug = 0;
   96 #else
   97 #define DPRINTF(x)
   98 #define DPRINTFN(n,x)
   99 #endif
  100 
  101 
  102 #define SA_LWP_STATE_LOCK(l, f) do {                            \
  103         (f) = (l)->l_flag;                                      \
  104         (l)->l_flag &= ~L_SA;                                   \
  105 } while (/*CONSTCOND*/ 0)
  106 
  107 #define SA_LWP_STATE_UNLOCK(l, f) do {                          \
  108         (l)->l_flag |= (f) & L_SA;                              \
  109 } while (/*CONSTCOND*/ 0)
  110 
  111 SPLAY_PROTOTYPE(sasttree, sastack, sast_node, sast_compare);
  112 SPLAY_GENERATE(sasttree, sastack, sast_node, sast_compare);
  113 
  114 /*
  115  * sa_critpath API
  116  * permit other parts of the kernel to make SA_LWP_STATE_{UN,}LOCK calls.
  117  */
  118 void
  119 sa_critpath_enter(struct lwp *l1, sa_critpath_t *f1)
  120 {
  121         SA_LWP_STATE_LOCK(l1, *f1);
  122 }
  123 void
  124 sa_critpath_exit(struct lwp *l1, sa_critpath_t *f1)
  125 {
  126         SA_LWP_STATE_UNLOCK(l1, *f1);
  127 }
  128 
  129 
  130 /*
  131  * sadata_upcall_alloc:
  132  *
  133  *      Allocate an sadata_upcall structure.
  134  */
  135 struct sadata_upcall *
  136 sadata_upcall_alloc(int waitok)
  137 {
  138         struct sadata_upcall *sau;
  139 
  140         sau = pool_get(&saupcall_pool, waitok ? PR_WAITOK : PR_NOWAIT);
  141         if (sau) {
  142                 sau->sau_arg = NULL;
  143         }
  144         return sau;
  145 }
  146 
  147 /*
  148  * sadata_upcall_free:
  149  *
  150  *      Free an sadata_upcall structure and any associated argument data.
  151  */
  152 void
  153 sadata_upcall_free(struct sadata_upcall *sau)
  154 {
  155 
  156         if (sau == NULL) {
  157                 return;
  158         }
  159         if (sau->sau_arg) {
  160                 (*sau->sau_argfreefunc)(sau->sau_arg);
  161         }
  162         pool_put(&saupcall_pool, sau);
  163 }
  164 
  165 /*
  166  * sa_newsavp
  167  *
  168  * Allocate a new virtual processor structure, do some simple
  169  * initialization and add it to the passed-in sa. Pre-allocate
  170  * an upcall event data structure for when the main thread on
  171  * this vp blocks.
  172  *
  173  * We lock sa_lock while manipulating the list of vp's.
  174  *
  175  * We allocate the lwp to run on this separately. In the case of the
  176  * first lwp/vp for a process, the lwp already exists. It's the
  177  * main (only) lwp of the process.
  178  */
  179 static struct sadata_vp *
  180 sa_newsavp(struct sadata *sa)
  181 {
  182         struct sadata_vp *vp, *qvp;
  183         struct sadata_upcall *sau;
  184 
  185         /* Allocate virtual processor data structure */
  186         vp = pool_get(&savp_pool, PR_WAITOK);
  187         /* And preallocate an upcall data structure for sleeping */
  188         sau = sadata_upcall_alloc(1);
  189         /* Initialize. */
  190         memset(vp, 0, sizeof(*vp));
  191         simple_lock_init(&vp->savp_lock);
  192         vp->savp_lwp = NULL;
  193         vp->savp_wokenq_head = NULL;
  194         vp->savp_faultaddr = 0;
  195         vp->savp_ofaultaddr = 0;
  196         LIST_INIT(&vp->savp_lwpcache);
  197         vp->savp_ncached = 0;
  198         vp->savp_sleeper_upcall = sau;
  199         SIMPLEQ_INIT(&vp->savp_upcalls);
  200 
  201         simple_lock(&sa->sa_lock);
  202         /* find first free savp_id and add vp to sorted slist */
  203         if (SLIST_EMPTY(&sa->sa_vps) ||
  204             SLIST_FIRST(&sa->sa_vps)->savp_id != 0) {
  205                 vp->savp_id = 0;
  206                 SLIST_INSERT_HEAD(&sa->sa_vps, vp, savp_next);
  207         } else {
  208                 SLIST_FOREACH(qvp, &sa->sa_vps, savp_next) {
  209                         if (SLIST_NEXT(qvp, savp_next) == NULL ||
  210                             SLIST_NEXT(qvp, savp_next)->savp_id !=
  211                             qvp->savp_id + 1)
  212                                 break;
  213                 }
  214                 vp->savp_id = qvp->savp_id + 1;
  215                 SLIST_INSERT_AFTER(qvp, vp, savp_next);
  216         }
  217         simple_unlock(&sa->sa_lock);
  218 
  219         return (vp);
  220 }
  221 
  222 /*
  223  * sys_sa_register
  224  *      Handle copyin and copyout of info for registering the
  225  * upcall handler address.
  226  */
  227 int
  228 sys_sa_register(struct lwp *l, void *v, register_t *retval)
  229 {
  230         struct sys_sa_register_args /* {
  231                 syscallarg(sa_upcall_t) new;
  232                 syscallarg(sa_upcall_t *) old;
  233                 syscallarg(int) flags;
  234                 syscallarg(ssize_t) stackinfo_offset;
  235         } */ *uap = v;
  236         int error;
  237         sa_upcall_t prev;
  238 
  239         error = dosa_register(l, SCARG(uap, new), &prev, SCARG(uap, flags),
  240             SCARG(uap, stackinfo_offset));
  241         if (error)
  242                 return error;
  243 
  244         if (SCARG(uap, old))
  245                 return copyout(&prev, SCARG(uap, old),
  246                     sizeof(prev));
  247         return 0;
  248 }
  249 
  250 /*
  251  * dosa_register
  252  *
  253  *      Change the upcall address for the process. If needed, allocate
  254  * an sadata structure (and initialize it) for the process. If initializing,
  255  * set the flags in the sadata structure to those passed in. Flags will
  256  * be ignored if the sadata structure already exists (dosa_regiister was
  257  * already called).
  258  *
  259  * Note: changing the upcall handler address for a process that has
  260  * concurrency greater than one can yield ambiguous results. The one
  261  * guarantee we can offer is that any upcalls generated on all CPUs
  262  * after this routine finishes will use the new upcall handler. Note
  263  * that any upcalls delivered upon return to user level by the
  264  * sys_sa_register() system call that called this routine will use the
  265  * new upcall handler. Note that any such upcalls will be delivered
  266  * before the old upcall handling address has been returned to
  267  * the application.
  268  */
  269 int
  270 dosa_register(struct lwp *l, sa_upcall_t new, sa_upcall_t *prev, int flags,
  271     ssize_t stackinfo_offset)
  272 {
  273         struct proc *p = l->l_proc;
  274         struct sadata *sa;
  275 
  276         if (p->p_sa == NULL) {
  277                 /* Allocate scheduler activations data structure */
  278                 sa = pool_get(&sadata_pool, PR_WAITOK);
  279                 /* Initialize. */
  280                 memset(sa, 0, sizeof(*sa));
  281                 simple_lock_init(&sa->sa_lock);
  282                 sa->sa_flag = flags & SA_FLAG_ALL;
  283                 sa->sa_maxconcurrency = 1;
  284                 sa->sa_concurrency = 1;
  285                 SPLAY_INIT(&sa->sa_stackstree);
  286                 sa->sa_stacknext = NULL;
  287                 if (flags & SA_FLAG_STACKINFO)
  288                         sa->sa_stackinfo_offset = stackinfo_offset;
  289                 else
  290                         sa->sa_stackinfo_offset = 0;
  291                 sa->sa_nstacks = 0;
  292                 SLIST_INIT(&sa->sa_vps);
  293                 p->p_sa = sa;
  294                 KASSERT(l->l_savp == NULL);
  295         }
  296         if (l->l_savp == NULL) {
  297                 l->l_savp = sa_newsavp(p->p_sa);
  298                 sa_newcachelwp(l);
  299         }
  300 
  301         *prev = p->p_sa->sa_upcall;
  302         p->p_sa->sa_upcall = new;
  303 
  304         return (0);
  305 }
  306 
  307 void
  308 sa_release(struct proc *p)
  309 {
  310         struct sadata *sa;
  311         struct sastack *sast, *next;
  312         struct sadata_vp *vp;
  313         struct lwp *l;
  314 
  315         sa = p->p_sa;
  316         KDASSERT(sa != NULL);
  317         KASSERT(p->p_nlwps <= 1);
  318 
  319         for (sast = SPLAY_MIN(sasttree, &sa->sa_stackstree); sast != NULL;
  320              sast = next) {
  321                 next = SPLAY_NEXT(sasttree, &sa->sa_stackstree, sast);
  322                 SPLAY_REMOVE(sasttree, &sa->sa_stackstree, sast);
  323                 pool_put(&sastack_pool, sast);
  324         }
  325 
  326         p->p_flag &= ~P_SA;
  327         while ((vp = SLIST_FIRST(&p->p_sa->sa_vps)) != NULL) {
  328                 SLIST_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next);
  329                 if (vp->savp_sleeper_upcall) {
  330                         sadata_upcall_free(vp->savp_sleeper_upcall);
  331                         vp->savp_sleeper_upcall = NULL;
  332                 }
  333                 pool_put(&savp_pool, vp);
  334         }
  335         pool_put(&sadata_pool, sa);
  336         p->p_sa = NULL;
  337         l = LIST_FIRST(&p->p_lwps);
  338         if (l) {
  339                 KASSERT(LIST_NEXT(l, l_sibling) == NULL);
  340                 l->l_savp = NULL;
  341         }
  342 }
  343 
  344 /*
  345  * sa_fetchstackgen
  346  *
  347  *      copyin the generation number for the stack in question.
  348  *
  349  * WRS: I think this routine needs the SA_LWP_STATE_LOCK() dance, either
  350  * here or in its caller.
  351  */
  352 static int
  353 sa_fetchstackgen(struct sastack *sast, struct sadata *sa, unsigned int *gen)
  354 {
  355         int error;
  356 
  357         /* COMPAT_NETBSD32:  believe it or not, but the following is ok */
  358         error = copyin(&((struct sa_stackinfo_t *)
  359             ((char *)sast->sast_stack.ss_sp +
  360             sa->sa_stackinfo_offset))->sasi_stackgen, gen, sizeof(*gen));
  361 
  362         return error;
  363 }
  364 
  365 /*
  366  * sa_stackused
  367  *
  368  *      Convenience routine to determine if a given stack has been used
  369  * or not. We consider a stack to be unused if the kernel's concept
  370  * of its generation number matches that of userland.
  371  *      We kill the application with SIGILL if there is an error copying
  372  * in the userland generation number.
  373  */
  374 static inline int
  375 sa_stackused(struct sastack *sast, struct sadata *sa)
  376 {
  377         unsigned int gen;
  378 
  379         if (sa_fetchstackgen(sast, sa, &gen)) {
  380 #ifdef DIAGNOSTIC
  381                 printf("sa_stackused: couldn't copyin sasi_stackgen");
  382 #endif
  383                 sigexit(curlwp, SIGILL);
  384                 /* NOTREACHED */
  385         }
  386         return (sast->sast_gen != gen);
  387 }
  388 
  389 /*
  390  * sa_setstackfree
  391  *
  392  *      Convenience routine to mark a stack as unused in the kernel's
  393  * eyes. We do this by setting the kernel's generation number for the stack
  394  * to that of userland.
  395  *      We kill the application with SIGILL if there is an error copying
  396  * in the userland generation number.
  397  */
  398 static inline void
  399 sa_setstackfree(struct sastack *sast, struct sadata *sa)
  400 {
  401         unsigned int gen;
  402 
  403         if (sa_fetchstackgen(sast, sa, &gen)) {
  404 #ifdef DIAGNOSTIC
  405                 printf("sa_setstackfree: couldn't copyin sasi_stackgen");
  406 #endif
  407                 sigexit(curlwp, SIGILL);
  408                 /* NOTREACHED */
  409         }
  410         sast->sast_gen = gen;
  411 }
  412 
  413 /*
  414  * sa_getstack
  415  *
  416  * Find next free stack, starting at sa->sa_stacknext.
  417  *
  418  * Caller must have the splay tree locked and should have cleared L_SA for
  419  * our thread. This is not the time to go generating upcalls as we aren't
  420  * in a position to deliver another one.
  421  */
  422 static struct sastack *
  423 sa_getstack(struct sadata *sa)
  424 {
  425         struct sastack *sast;
  426 
  427         SCHED_ASSERT_UNLOCKED();
  428 
  429         if ((sast = sa->sa_stacknext) == NULL || sa_stackused(sast, sa))
  430                 sast = sa_getstack0(sa);
  431 
  432         if (sast == NULL)
  433                 return NULL;
  434 
  435         sast->sast_gen++;
  436 
  437         return sast;
  438 }
  439 
  440 /*
  441  * sa_getstack0 -- get the lowest numbered sa stack
  442  *
  443  *      We walk the splay tree in order and find the lowest-numbered
  444  * (as defined by SPLAY_MIN() and SPLAY_NEXT() ordering) stack that
  445  * is unused.
  446  */
  447 static inline struct sastack *
  448 sa_getstack0(struct sadata *sa)
  449 {
  450         struct sastack *start;
  451 
  452         if (sa->sa_stacknext == NULL) {
  453                 sa->sa_stacknext = SPLAY_MIN(sasttree, &sa->sa_stackstree);
  454                 if (sa->sa_stacknext == NULL)
  455                         return NULL;
  456         }
  457         start = sa->sa_stacknext;
  458 
  459         while (sa_stackused(sa->sa_stacknext, sa)) {
  460                 sa->sa_stacknext = SPLAY_NEXT(sasttree, &sa->sa_stackstree,
  461                     sa->sa_stacknext);
  462                 if (sa->sa_stacknext == NULL)
  463                         sa->sa_stacknext = SPLAY_MIN(sasttree,
  464                             &sa->sa_stackstree);
  465                 if (sa->sa_stacknext == start)
  466                         return NULL;
  467         }
  468         return sa->sa_stacknext;
  469 }
  470 
  471 /*
  472  * sast_compare - compare two sastacks
  473  *
  474  *      We sort stacks according to their userspace addresses.
  475  * Stacks are "equal" if their start + size overlap.
  476  */
  477 static inline int
  478 sast_compare(struct sastack *a, struct sastack *b)
  479 {
  480         if ((vaddr_t)a->sast_stack.ss_sp + a->sast_stack.ss_size <=
  481             (vaddr_t)b->sast_stack.ss_sp)
  482                 return (-1);
  483         if ((vaddr_t)a->sast_stack.ss_sp >=
  484             (vaddr_t)b->sast_stack.ss_sp + b->sast_stack.ss_size)
  485                 return (1);
  486         return (0);
  487 }
  488 
  489 /*
  490  * sa_copyin_stack -- copyin a stack.
  491  */
  492 static int
  493 sa_copyin_stack(stack_t *stacks, int index, stack_t *dest)
  494 {
  495         return copyin(stacks + index, dest, sizeof(stack_t));
  496 }
  497 
  498 /*
  499  * sys_sa_stacks -- the user level threading library is passing us stacks
  500  *
  501  * We copy in some arguments then call sa_stacks1() to do the main
  502  * work. NETBSD32 has its own front-end for this call.
  503  */
  504 int
  505 sys_sa_stacks(struct lwp *l, void *v, register_t *retval)
  506 {
  507         struct sys_sa_stacks_args /* {
  508                 syscallarg(int) num;
  509                 syscallarg(stack_t *) stacks;
  510         } */ *uap = v;
  511 
  512         return sa_stacks1(l, retval, SCARG(uap, num), SCARG(uap, stacks), sa_copyin_stack);
  513 }
  514 
  515 /*
  516  * sa_stacks1
  517  *      Process stacks passed-in by the user threading library. At
  518  * present we use the kernel lock to lock the SPLAY tree, which we
  519  * manipulate to load in the stacks.
  520  *
  521  *      It is an error to pass in a stack that we already know about
  522  * and which hasn't been used. Passing in a known-but-used one is fine.
  523  * We accept up to SA_MAXNUMSTACKS per desired vp (concurrency level).
  524  */
  525 int
  526 sa_stacks1(struct lwp *l, register_t *retval, int num, stack_t *stacks,
  527     sa_copyin_stack_t do_sa_copyin_stack)
  528 {
  529         struct sadata *sa = l->l_proc->p_sa;
  530         struct sastack *sast, newsast;
  531         int count, error, f, i;
  532 
  533         /* We have to be using scheduler activations */
  534         if (sa == NULL)
  535                 return (EINVAL);
  536 
  537         count = num;
  538         if (count < 0)
  539                 return (EINVAL);
  540 
  541         SA_LWP_STATE_LOCK(l, f);
  542 
  543         error = 0;
  544 
  545         for (i = 0; i < count; i++) {
  546                 error = do_sa_copyin_stack(stacks, i, &newsast.sast_stack);
  547                 if (error) {
  548                         count = i;
  549                         break;
  550                 }
  551                 sast = SPLAY_FIND(sasttree, &sa->sa_stackstree, &newsast);
  552                 if (sast != NULL) {
  553                         DPRINTFN(9, ("sa_stacks(%d.%d) returning stack %p\n",
  554                                      l->l_proc->p_pid, l->l_lid,
  555                                      newsast.sast_stack.ss_sp));
  556                         if (sa_stackused(sast, sa) == 0) {
  557                                 count = i;
  558                                 error = EEXIST;
  559                                 break;
  560                         }
  561                 } else if (sa->sa_nstacks >=
  562                     SA_MAXNUMSTACKS * sa->sa_concurrency) {
  563                         DPRINTFN(9,
  564                             ("sa_stacks(%d.%d) already using %d stacks\n",
  565                             l->l_proc->p_pid, l->l_lid,
  566                             SA_MAXNUMSTACKS * sa->sa_concurrency));
  567                         count = i;
  568                         error = ENOMEM;
  569                         break;
  570                 } else {
  571                         DPRINTFN(9, ("sa_stacks(%d.%d) adding stack %p\n",
  572                                      l->l_proc->p_pid, l->l_lid,
  573                                      newsast.sast_stack.ss_sp));
  574                         sast = pool_get(&sastack_pool, PR_WAITOK);
  575                         sast->sast_stack = newsast.sast_stack;
  576                         SPLAY_INSERT(sasttree, &sa->sa_stackstree, sast);
  577                         sa->sa_nstacks++;
  578                 }
  579                 sa_setstackfree(sast, sa);
  580         }
  581 
  582         SA_LWP_STATE_UNLOCK(l, f);
  583 
  584         *retval = count;
  585         return (error);
  586 }
  587 
  588 
  589 /*
  590  * sys_sa_enable - throw the switch & enable SA
  591  *
  592  * Fairly simple. Make sure the sadata and vp've been set up for this
  593  * process, assign this thread to the vp and initiate the first upcall
  594  * (SA_UPCALL_NEWPROC).
  595  */
  596 int
  597 sys_sa_enable(struct lwp *l, void *v, register_t *retval)
  598 {
  599         struct proc *p = l->l_proc;
  600         struct sadata *sa = p->p_sa;
  601         struct sadata_vp *vp = l->l_savp;
  602         int error;
  603 
  604         DPRINTF(("sys_sa_enable(%d.%d)\n", l->l_proc->p_pid,
  605             l->l_lid));
  606 
  607         /* We have to be using scheduler activations */
  608         if (sa == NULL || vp == NULL)
  609                 return (EINVAL);
  610 
  611         if (p->p_flag & P_SA) /* Already running! */
  612                 return (EBUSY);
  613 
  614         error = sa_upcall(l, SA_UPCALL_NEWPROC, l, NULL, 0, NULL, NULL);
  615         if (error)
  616                 return (error);
  617 
  618         /* Assign this LWP to the virtual processor */
  619         vp->savp_lwp = l;
  620 
  621         p->p_flag |= P_SA;
  622         l->l_flag |= L_SA; /* We are now an activation LWP */
  623 
  624         /*
  625          * This will return to the SA handler previously registered.
  626          */
  627         return (0);
  628 }
  629 
  630 
  631 /*
  632  * sa_increaseconcurrency
  633  *      Raise the process's maximum concurrency level to the
  634  * requested level. Does nothing if the current maximum councurrency
  635  * is greater than the requested.
  636  *      Uses the kernel lock to implicitly lock operations. Also
  637  * uses sa_lock to lock the vp list and uses SCHED_LOCK() to lock the
  638  * scheduler.
  639  */
  640 #ifdef MULTIPROCESSOR
  641 static int
  642 sa_increaseconcurrency(struct lwp *l, int concurrency)
  643 {
  644         struct proc *p;
  645         struct lwp *l2;
  646         struct sadata *sa;
  647         vaddr_t uaddr;
  648         boolean_t inmem;
  649         int addedconcurrency, error, s;
  650         struct sadata_vp *vp;
  651 
  652         p = l->l_proc;
  653         sa = p->p_sa;
  654 
  655         addedconcurrency = 0;
  656         simple_lock(&sa->sa_lock);
  657         while (sa->sa_maxconcurrency < concurrency) {
  658                 sa->sa_maxconcurrency++;
  659                 sa->sa_concurrency++;
  660                 simple_unlock(&sa->sa_lock);
  661 
  662                 inmem = uvm_uarea_alloc(&uaddr);
  663                 if (__predict_false(uaddr == 0)) {
  664                         /* reset concurrency */
  665                         simple_lock(&sa->sa_lock);
  666                         sa->sa_maxconcurrency--;
  667                         sa->sa_concurrency--;
  668                         simple_unlock(&sa->sa_lock);
  669                         return (addedconcurrency);
  670                 } else {
  671                         newlwp(l, p, uaddr, inmem, 0, NULL, 0,
  672                             child_return, 0, &l2);
  673                         l2->l_flag |= L_SA;
  674                         l2->l_savp = vp = sa_newsavp(sa);
  675                         if (vp) {
  676                                 vp->savp_lwp = l2;
  677                                 cpu_setfunc(l2, sa_switchcall, NULL);
  678                                 error = sa_upcall(l2, SA_UPCALL_NEWPROC,
  679                                     NULL, NULL, 0, NULL, NULL);
  680                                 if (error) {
  681                                         /* free new savp */
  682                                         SLIST_REMOVE(&sa->sa_vps, vp,
  683                                             sadata_vp, savp_next);
  684                                         if (vp->savp_sleeper_upcall) {
  685                                                 sadata_upcall_free(
  686                                                     vp->savp_sleeper_upcall);
  687                                                 vp->savp_sleeper_upcall = NULL;
  688                                         }
  689                                         pool_put(&savp_pool, vp);
  690                                 }
  691                         } else
  692                                 error = 1;
  693                         if (error) {
  694                                 /* put l2 into l's VP LWP cache */
  695                                 l2->l_savp = l->l_savp;
  696                                 PHOLD(l2);
  697                                 SCHED_LOCK(s);
  698                                 sa_putcachelwp(p, l2);
  699                                 SCHED_UNLOCK(s);
  700                                 /* reset concurrency */
  701                                 simple_lock(&sa->sa_lock);
  702                                 sa->sa_maxconcurrency--;
  703                                 sa->sa_concurrency--;
  704                                 simple_unlock(&sa->sa_lock);
  705                                 return (addedconcurrency);
  706                         }
  707                         SCHED_LOCK(s);
  708                         setrunnable(l2);
  709                         SCHED_UNLOCK(s);
  710                         addedconcurrency++;
  711                 }
  712                 simple_lock(&sa->sa_lock);
  713         }
  714         simple_unlock(&sa->sa_lock);
  715 
  716         return (addedconcurrency);
  717 }
  718 #endif
  719 
  720 /*
  721  * sys_sa_setconcurrency
  722  *      The user threading library wants to increase the number
  723  * of active virtual CPUS we assign to it. We return the number of virt
  724  * CPUs we assigned to the process. We limit concurrency to the number
  725  * of CPUs in the system.
  726  *
  727  * WRS: at present, this system call serves two purposes. The first is
  728  * for an application to indicate that it wants a certain concurrency
  729  * level. The second is for the application to request that the kernel
  730  * reactivate previously allocated virtual CPUs.
  731  */
  732 int
  733 sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval)
  734 {
  735         struct sys_sa_setconcurrency_args /* {
  736                 syscallarg(int) concurrency;
  737         } */ *uap = v;
  738         struct sadata *sa = l->l_proc->p_sa;
  739 #ifdef MULTIPROCESSOR
  740         struct sadata_vp *vp = l->l_savp;
  741         int ncpus, s;
  742         struct cpu_info *ci;
  743         CPU_INFO_ITERATOR cii;
  744 #endif
  745 
  746         DPRINTFN(11,("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid,
  747                      l->l_lid));
  748 
  749         /* We have to be using scheduler activations */
  750         if (sa == NULL)
  751                 return (EINVAL);
  752 
  753         if ((l->l_proc->p_flag & P_SA) == 0)
  754                 return (EINVAL);
  755 
  756         if (SCARG(uap, concurrency) < 1)
  757                 return (EINVAL);
  758 
  759         *retval = 0;
  760         /*
  761          * Concurrency greater than the number of physical CPUs does
  762          * not make sense.
  763          * XXX Should we ever support hot-plug CPUs, this will need
  764          * adjustment.
  765          */
  766 #ifdef MULTIPROCESSOR
  767         if (SCARG(uap, concurrency) > sa->sa_maxconcurrency) {
  768                 ncpus = 0;
  769                 for (CPU_INFO_FOREACH(cii, ci))
  770                         ncpus++;
  771                 *retval += sa_increaseconcurrency(l,
  772                     min(SCARG(uap, concurrency), ncpus));
  773         }
  774 #endif
  775 
  776         DPRINTFN(11,("sys_sa_concurrency(%d.%d) want %d, have %d, max %d\n",
  777                      l->l_proc->p_pid, l->l_lid, SCARG(uap, concurrency),
  778                      sa->sa_concurrency, sa->sa_maxconcurrency));
  779 #ifdef MULTIPROCESSOR
  780         if (SCARG(uap, concurrency) > sa->sa_concurrency) {
  781                 SCHED_LOCK(s);
  782                 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) {
  783                         if (vp->savp_lwp->l_flag & L_SA_IDLE) {
  784                                 vp->savp_lwp->l_flag &=
  785                                         ~(L_SA_IDLE|L_SA_YIELD|L_SINTR);
  786                                 SCHED_UNLOCK(s);
  787                                 DPRINTFN(11,("sys_sa_concurrency(%d.%d) "
  788                                              "NEWPROC vp %d\n",
  789                                              l->l_proc->p_pid, l->l_lid,
  790                                              vp->savp_id));
  791                                 cpu_setfunc(vp->savp_lwp, sa_switchcall, NULL);
  792                                 /* error = */ sa_upcall(vp->savp_lwp,
  793                                     SA_UPCALL_NEWPROC,
  794                                     NULL, NULL, 0, NULL, NULL);
  795                                 SCHED_LOCK(s);
  796                                 sa->sa_concurrency++;
  797                                 setrunnable(vp->savp_lwp);
  798                                 KDASSERT((vp->savp_lwp->l_flag & L_SINTR) == 0);
  799                                 (*retval)++;
  800                         }
  801                         if (sa->sa_concurrency == SCARG(uap, concurrency))
  802                                 break;
  803                 }
  804                 SCHED_UNLOCK(s);
  805         }
  806 #endif
  807 
  808         return (0);
  809 }
  810 
  811 /*
  812  * sys_sa_yield
  813  *      application has nothing for this lwp to do, so let it linger in
  814  * the kernel.
  815  */
  816 int
  817 sys_sa_yield(struct lwp *l, void *v, register_t *retval)
  818 {
  819         struct proc *p = l->l_proc;
  820 
  821         if (p->p_sa == NULL || !(p->p_flag & P_SA)) {
  822                 DPRINTFN(1,
  823                     ("sys_sa_yield(%d.%d) proc %p not SA (p_sa %p, flag %s)\n",
  824                     p->p_pid, l->l_lid, p, p->p_sa,
  825                     p->p_flag & P_SA ? "T" : "F"));
  826                 return (EINVAL);
  827         }
  828 
  829         sa_yield(l);
  830 
  831         return (EJUSTRETURN);
  832 }
  833 
  834 /*
  835  * sa_yield
  836  *      This lwp has nothing to do, so hang around. Assuming we
  837  * are the lwp "on" our vp, tsleep in "sawait" until there's something
  838  * to do. If there are upcalls, we deliver them explicitly.
  839  */
  840 void
  841 sa_yield(struct lwp *l)
  842 {
  843         struct proc *p = l->l_proc;
  844         struct sadata *sa = p->p_sa;
  845         struct sadata_vp *vp = l->l_savp;
  846         int ret;
  847 
  848         KERNEL_LOCK_ASSERT_LOCKED();
  849 
  850         if (vp->savp_lwp != l) {
  851                 /*
  852                  * We lost the VP on our way here, this happens for
  853                  * instance when we sleep in systrace.  This will end
  854                  * in an SA_UNBLOCKED_UPCALL in sa_setwoken().
  855                  */
  856                 DPRINTFN(1,("sa_yield(%d.%d) lost VP\n",
  857                              p->p_pid, l->l_lid));
  858                 KDASSERT(l->l_flag & L_SA_BLOCKING);
  859                 return;
  860         }
  861 
  862         /*
  863          * If we're the last running LWP, stick around to receive
  864          * signals.
  865          */
  866         KDASSERT((l->l_flag & L_SA_YIELD) == 0);
  867         DPRINTFN(1,("sa_yield(%d.%d) going dormant\n",
  868                      p->p_pid, l->l_lid));
  869         /*
  870          * A signal will probably wake us up. Worst case, the upcall
  871          * happens and just causes the process to yield again.
  872          */
  873         /* s = splsched(); */   /* Protect from timer expirations */
  874         KDASSERT(vp->savp_lwp == l);
  875         /*
  876          * If we were told to make an upcall or exit before
  877          * the splsched(), make sure we process it instead of
  878          * going to sleep. It might make more sense for this to
  879          * be handled inside of tsleep....
  880          */
  881         ret = 0;
  882         l->l_flag |= L_SA_YIELD;
  883         if (l->l_flag & L_SA_UPCALL) {
  884                 /* KERNEL_PROC_UNLOCK(l); in upcallret() */
  885                 upcallret(l);
  886                 KERNEL_PROC_LOCK(l);
  887         }
  888         while (l->l_flag & L_SA_YIELD) {
  889                 DPRINTFN(1,("sa_yield(%d.%d) really going dormant\n",
  890                              p->p_pid, l->l_lid));
  891 
  892                 simple_lock(&sa->sa_lock);
  893                 sa->sa_concurrency--;
  894                 simple_unlock(&sa->sa_lock);
  895 
  896                 ret = tsleep(l, PUSER | PCATCH, "sawait", 0);
  897 
  898                 simple_lock(&sa->sa_lock);
  899                 sa->sa_concurrency++;
  900                 simple_unlock(&sa->sa_lock);
  901 
  902                 KDASSERT(vp->savp_lwp == l || p->p_flag & P_WEXIT);
  903 
  904                 /* KERNEL_PROC_UNLOCK(l); in upcallret() */
  905                 upcallret(l);
  906                 KERNEL_PROC_LOCK(l);
  907         }
  908         /* splx(s); */
  909         DPRINTFN(1,("sa_yield(%d.%d) returned, ret %d, userret %p\n",
  910                      p->p_pid, l->l_lid, ret, p->p_userret));
  911 }
  912 
  913 
  914 /*
  915  * sys_sa_preempt - preempt a running thread
  916  *
  917  * Given an lwp id, send it a user upcall. This is a way for libpthread to
  918  * kick something into the upcall handler.
  919  */
  920 int
  921 sys_sa_preempt(struct lwp *l, void *v, register_t *retval)
  922 {
  923         struct sys_sa_preempt_args /* {
  924                 syscallarg(int) sa_id;
  925         } */ *uap = v;
  926         struct sadata           *sa = l->l_proc->p_sa;
  927         struct lwp              *t;
  928         int                     target, s, error;
  929 
  930         DPRINTFN(11,("sys_sa_preempt(%d.%d)\n", l->l_proc->p_pid,
  931                      l->l_lid));
  932 
  933         /* We have to be using scheduler activations */
  934         if (sa == NULL)
  935                 return (EINVAL);
  936 
  937         if ((l->l_proc->p_flag & P_SA) == 0)
  938                 return (EINVAL);
  939 
  940         if ((target = SCARG(uap, sa_id)) < 1)
  941                 return (EINVAL);
  942 
  943         SCHED_LOCK(s);
  944 
  945         LIST_FOREACH(t, &l->l_proc->p_lwps, l_sibling)
  946                 if (t->l_lid == target)
  947                         break;
  948 
  949         if (t == NULL) {
  950                 error = ESRCH;
  951                 goto exit_lock;
  952         }
  953 
  954         /* XXX WRS We really need all of this locking documented */
  955         SCHED_UNLOCK(s);
  956 
  957         error = sa_upcall(l, SA_UPCALL_USER | SA_UPCALL_DEFER_EVENT, l, NULL,
  958                 0, NULL, NULL);
  959         if (error)
  960                 return error;
  961 
  962         return 0;
  963 
  964 exit_lock:
  965         SCHED_UNLOCK(s);
  966 
  967         return error;
  968 }
  969 
  970 
  971 /* XXX Hm, naming collision. */
  972 void
  973 sa_preempt(struct lwp *l)
  974 {
  975         struct proc *p = l->l_proc;
  976         struct sadata *sa = p->p_sa;
  977 
  978         /*
  979          * Defer saving the lwp's state because on some ports
  980          * preemption can occur between generating an unblocked upcall
  981          * and processing the upcall queue.
  982          */
  983         if (sa->sa_flag & SA_FLAG_PREEMPT)
  984                 sa_upcall(l, SA_UPCALL_PREEMPTED | SA_UPCALL_DEFER_EVENT,
  985                     l, NULL, 0, NULL, NULL);
  986 }
  987 
  988 
  989 /*
  990  * Set up the user-level stack and trapframe to do an upcall.
  991  *
  992  * NOTE: This routine WILL FREE "arg" in the case of failure!  Callers
  993  * should not touch the "arg" pointer once calling sa_upcall().
  994  */
  995 int
  996 sa_upcall(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted,
  997         size_t argsize, void *arg, void (*func)(void *))
  998 {
  999         struct sadata_upcall *sau;
 1000         struct sadata *sa = l->l_proc->p_sa;
 1001         struct sadata_vp *vp = l->l_savp;
 1002         struct sastack *sast;
 1003         int f, error;
 1004 
 1005         /* XXX prevent recursive upcalls if we sleep for memory */
 1006         SA_LWP_STATE_LOCK(l, f);
 1007         sast = sa_getstack(sa);
 1008         SA_LWP_STATE_UNLOCK(l, f);
 1009         if (sast == NULL) {
 1010                 return (ENOMEM);
 1011         }
 1012         DPRINTFN(9,("sa_upcall(%d.%d) using stack %p\n",
 1013             l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
 1014 
 1015         if (l->l_proc->p_emul->e_sa->sae_upcallconv) {
 1016                 error = (*l->l_proc->p_emul->e_sa->sae_upcallconv)(l, type,
 1017                     &argsize, &arg, &func);
 1018                 if (error)
 1019                         return error;
 1020         }
 1021 
 1022         SA_LWP_STATE_LOCK(l, f);
 1023         sau = sadata_upcall_alloc(1);
 1024         SA_LWP_STATE_UNLOCK(l, f);
 1025         sa_upcall0(sau, type, event, interrupted, argsize, arg, func);
 1026         sau->sau_stack = sast->sast_stack;
 1027 
 1028         SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
 1029         l->l_flag |= L_SA_UPCALL;
 1030 
 1031         return (0);
 1032 }
 1033 
 1034 static void
 1035 sa_upcall0(struct sadata_upcall *sau, int type, struct lwp *event,
 1036     struct lwp *interrupted, size_t argsize, void *arg, void (*func)(void *))
 1037 {
 1038 
 1039         KDASSERT((event == NULL) || (event != interrupted));
 1040 
 1041         sau->sau_flags = 0;
 1042 
 1043         if (type & SA_UPCALL_DEFER_EVENT) {
 1044                 sau->sau_event.ss_deferred.ss_lwp = event;
 1045                 sau->sau_flags |= SAU_FLAG_DEFERRED_EVENT;
 1046         } else
 1047                 sa_upcall_getstate(&sau->sau_event, event);
 1048         if (type & SA_UPCALL_DEFER_INTERRUPTED) {
 1049                 sau->sau_interrupted.ss_deferred.ss_lwp = interrupted;
 1050                 sau->sau_flags |= SAU_FLAG_DEFERRED_INTERRUPTED;
 1051         } else
 1052                 sa_upcall_getstate(&sau->sau_interrupted, interrupted);
 1053 
 1054         sau->sau_type = type & SA_UPCALL_TYPE_MASK;
 1055         sau->sau_argsize = argsize;
 1056         sau->sau_arg = arg;
 1057         sau->sau_argfreefunc = func;
 1058 }
 1059 
 1060 /*
 1061  * sa_ucsp
 1062  *      return the stack pointer (??) for a given context as
 1063  * reported by the _UC_MACHINE_SP() macro.
 1064  */
 1065 void *
 1066 sa_ucsp(void *arg)
 1067 {
 1068         ucontext_t *uc = arg;
 1069 
 1070         return (void *)(uintptr_t)_UC_MACHINE_SP(uc);
 1071 }
 1072 
 1073 /*
 1074  * sa_upcall_getstate
 1075  *      Fill in the given sau_state with info for the passed-in
 1076  * lwp, and update the lwp accordingly.
 1077  * WRS: unwaware of any locking before entry!
 1078  *      We set L_SA_SWITCHING on the target lwp. We assume that l_flag is
 1079  * protected by the kernel lock and untouched in interrupt context.
 1080  */
 1081 static void
 1082 sa_upcall_getstate(union sau_state *ss, struct lwp *l)
 1083 {
 1084         caddr_t sp;
 1085         size_t ucsize;
 1086 
 1087         if (l) {
 1088                 l->l_flag |= L_SA_SWITCHING;
 1089                 (*l->l_proc->p_emul->e_sa->sae_getucontext)(l,
 1090                     (void *)&ss->ss_captured.ss_ctx);
 1091                 l->l_flag &= ~L_SA_SWITCHING;
 1092                 sp = (*l->l_proc->p_emul->e_sa->sae_ucsp)
 1093                     (&ss->ss_captured.ss_ctx);
 1094                 /* XXX COMPAT_NETBSD32: _UC_UCONTEXT_ALIGN */
 1095                 sp = STACK_ALIGN(sp, ~_UC_UCONTEXT_ALIGN);
 1096                 ucsize = roundup(l->l_proc->p_emul->e_sa->sae_ucsize,
 1097                     (~_UC_UCONTEXT_ALIGN) + 1);
 1098                 ss->ss_captured.ss_sa.sa_context =
 1099                     (ucontext_t *)STACK_ALLOC(sp, ucsize);
 1100                 ss->ss_captured.ss_sa.sa_id = l->l_lid;
 1101                 ss->ss_captured.ss_sa.sa_cpu = l->l_savp->savp_id;
 1102         } else
 1103                 ss->ss_captured.ss_sa.sa_context = NULL;
 1104 }
 1105 
 1106 
 1107 /*
 1108  * sa_pagefault
 1109  *
 1110  * Detect double pagefaults and pagefaults on upcalls.
 1111  * - double pagefaults are detected by comparing the previous faultaddr
 1112  *   against the current faultaddr
 1113  * - pagefaults on upcalls are detected by checking if the userspace
 1114  *   thread is running on an upcall stack
 1115  */
 1116 static inline int
 1117 sa_pagefault(struct lwp *l, ucontext_t *l_ctx)
 1118 {
 1119         struct proc *p;
 1120         struct sadata *sa;
 1121         struct sadata_vp *vp;
 1122         struct sastack sast;
 1123 
 1124         p = l->l_proc;
 1125         sa = p->p_sa;
 1126         vp = l->l_savp;
 1127 
 1128         KDASSERT(vp->savp_lwp == l);
 1129 
 1130         if (vp->savp_faultaddr == vp->savp_ofaultaddr) {
 1131                 DPRINTFN(10,("sa_pagefault(%d.%d) double page fault\n",
 1132                              p->p_pid, l->l_lid));
 1133                 return 1;
 1134         }
 1135 
 1136         sast.sast_stack.ss_sp = (*p->p_emul->e_sa->sae_ucsp)(l_ctx);
 1137         sast.sast_stack.ss_size = 1;
 1138 
 1139         if (SPLAY_FIND(sasttree, &sa->sa_stackstree, &sast)) {
 1140                 DPRINTFN(10,("sa_pagefault(%d.%d) upcall page fault\n",
 1141                              p->p_pid, l->l_lid));
 1142                 return 1;
 1143         }
 1144 
 1145         vp->savp_ofaultaddr = vp->savp_faultaddr;
 1146         return 0;
 1147 }
 1148 
 1149 
 1150 /*
 1151  * sa_switch
 1152  *
 1153  * Called by tsleep() when it wants to call mi_switch().
 1154  * Block current LWP and switch to another.
 1155  *
 1156  * WE ARE NOT ALLOWED TO SLEEP HERE!  WE ARE CALLED FROM WITHIN
 1157  * TSLEEP() ITSELF!  We are called with sched_lock held, and must
 1158  * hold it right through the mi_switch() call.
 1159  *
 1160  * We return with the scheduler unlocked.
 1161  *
 1162  * We are called in one of three conditions:
 1163  *
 1164  * 1:           We are an sa_yield thread. If there are any UNBLOCKED
 1165  *      upcalls to deliver, deliver them (by exiting) instead of sleeping.
 1166  * 2:           We are the main lwp (we're the lwp on our vp). Trigger
 1167  *      delivery of a BLOCKED upcall.
 1168  * 3:           We are not the main lwp on our vp. Chances are we got
 1169  *      woken up but the sleeper turned around and went back to sleep.
 1170  *      It seems that select and poll do this a lot. So just go back to sleep.
 1171  */
 1172 
 1173 void
 1174 sa_switch(struct lwp *l, int type)
 1175 {
 1176         struct proc *p = l->l_proc;
 1177         struct sadata_vp *vp = l->l_savp;
 1178         struct sadata_upcall *sau = NULL;
 1179         struct lwp *l2;
 1180         int s;
 1181 
 1182         DPRINTFN(4,("sa_switch(%d.%d type %d VP %d)\n", p->p_pid, l->l_lid,
 1183             type, vp->savp_lwp ? vp->savp_lwp->l_lid : 0));
 1184 
 1185         SCHED_ASSERT_LOCKED();
 1186 
 1187         if (p->p_flag & P_WEXIT) {
 1188                 mi_switch(l, NULL);
 1189                 return;
 1190         }
 1191 
 1192         if (l->l_flag & L_SA_YIELD) {
 1193 
 1194                 /*
 1195                  * Case 0: we're blocking in sa_yield
 1196                  */
 1197                 if (vp->savp_wokenq_head == NULL && p->p_userret == NULL) {
 1198                         l->l_flag |= L_SA_IDLE;
 1199                         mi_switch(l, NULL);
 1200                 } else {
 1201                         /* make us running again. */
 1202                         unsleep(l);
 1203                         l->l_stat = LSONPROC;
 1204                         l->l_proc->p_nrlwps++;
 1205                         s = splsched();
 1206                         SCHED_UNLOCK(s);
 1207                 }
 1208                 return;
 1209         } else if (vp->savp_lwp == l) {
 1210                 /*
 1211                  * Case 1: we're blocking for the first time; generate
 1212                  * a SA_BLOCKED upcall and allocate resources for the
 1213                  * UNBLOCKED upcall.
 1214                  */
 1215                 if (vp->savp_sleeper_upcall) {
 1216                         sau = vp->savp_sleeper_upcall;
 1217                         vp->savp_sleeper_upcall = NULL;
 1218                 }
 1219 
 1220                 if (sau == NULL) {
 1221 #ifdef DIAGNOSTIC
 1222                         printf("sa_switch(%d.%d): no upcall data.\n",
 1223                             p->p_pid, l->l_lid);
 1224 #endif
 1225 panic("Oops! Don't have a sleeper!\n");
 1226                         /* XXXWRS Shouldn't we just kill the app here? */
 1227                         mi_switch(l, NULL);
 1228                         return;
 1229                 }
 1230 
 1231                 /*
 1232                  * The process of allocating a new LWP could cause
 1233                  * sleeps. We're called from inside sleep, so that
 1234                  * would be Bad. Therefore, we must use a cached new
 1235                  * LWP. The first thing that this new LWP must do is
 1236                  * allocate another LWP for the cache.  */
 1237                 l2 = sa_getcachelwp(vp);
 1238                 if (l2 == NULL) {
 1239                         /* XXXSMP */
 1240                         /* No upcall for you! */
 1241                         /* XXX The consequences of this are more subtle and
 1242                          * XXX the recovery from this situation deserves
 1243                          * XXX more thought.
 1244                          */
 1245 
 1246                         /* XXXUPSXXX Should only happen with concurrency > 1 */
 1247 #ifdef DIAGNOSTIC
 1248                         printf("sa_switch(%d.%d): no cached LWP for upcall.\n",
 1249                             p->p_pid, l->l_lid);
 1250 #endif
 1251                         mi_switch(l, NULL);
 1252                         sadata_upcall_free(sau);
 1253                         return;
 1254                 }
 1255 
 1256                 cpu_setfunc(l2, sa_switchcall, sau);
 1257                 sa_upcall0(sau, SA_UPCALL_BLOCKED, l, NULL, 0, NULL, NULL);
 1258 
 1259                 /*
 1260                  * Perform the double/upcall pagefault check.
 1261                  * We do this only here since we need l's ucontext to
 1262                  * get l's userspace stack. sa_upcall0 above has saved
 1263                  * it for us.
 1264                  * The L_SA_PAGEFAULT flag is set in the MD
 1265                  * pagefault code to indicate a pagefault.  The MD
 1266                  * pagefault code also saves the faultaddr for us.
 1267                  */
 1268                 if ((l->l_flag & L_SA_PAGEFAULT) && sa_pagefault(l,
 1269                         &sau->sau_event.ss_captured.ss_ctx) != 0) {
 1270                         cpu_setfunc(l2, sa_switchcall, NULL);
 1271                         sa_putcachelwp(p, l2); /* PHOLD from sa_getcachelwp */
 1272                         mi_switch(l, NULL);
 1273                         /*
 1274                          * WRS Not sure how vp->savp_sleeper_upcall != NULL
 1275                          * but be careful none the less
 1276                          */
 1277                         if (vp->savp_sleeper_upcall == NULL)
 1278                                 vp->savp_sleeper_upcall = sau;
 1279                         else
 1280                                 sadata_upcall_free(sau);
 1281                         DPRINTFN(10,("sa_switch(%d.%d) page fault resolved\n",
 1282                                      p->p_pid, l->l_lid));
 1283                         if (vp->savp_faultaddr == vp->savp_ofaultaddr)
 1284                                 vp->savp_ofaultaddr = -1;
 1285                         return;
 1286                 }
 1287 
 1288                 DPRINTFN(8,("sa_switch(%d.%d) blocked upcall %d\n",
 1289                              p->p_pid, l->l_lid, l2->l_lid));
 1290 
 1291                 l->l_flag |= L_SA_BLOCKING;
 1292                 l2->l_priority = l2->l_usrpri;
 1293                 vp->savp_blocker = l;
 1294                 vp->savp_lwp = l2;
 1295                 setrunnable(l2);
 1296                 PRELE(l2); /* Remove the artificial hold-count */
 1297 
 1298                 KDASSERT(l2 != l);
 1299         } else if (vp->savp_lwp != NULL) {
 1300 
 1301                 /*
 1302                  * Case 2: We've been woken up while another LWP was
 1303                  * on the VP, but we're going back to sleep without
 1304                  * having returned to userland and delivering the
 1305                  * SA_UNBLOCKED upcall (select and poll cause this
 1306                  * kind of behavior a lot).
 1307                  */
 1308                 l2 = NULL;
 1309         } else {
 1310                 /* NOTREACHED */
 1311                 panic("sa_vp empty");
 1312         }
 1313 
 1314         DPRINTFN(4,("sa_switch(%d.%d) switching to LWP %d.\n",
 1315             p->p_pid, l->l_lid, l2 ? l2->l_lid : 0));
 1316         mi_switch(l, l2);
 1317         DPRINTFN(4,("sa_switch(%d.%d flag %x) returned.\n",
 1318             p->p_pid, l->l_lid, l->l_flag));
 1319         KDASSERT(l->l_wchan == 0);
 1320 
 1321         SCHED_ASSERT_UNLOCKED();
 1322 }
 1323 
 1324 /*
 1325  * sa_switchcall
 1326  *
 1327  * We need to pass an upcall to userland. We are now
 1328  * running on a spare stack and need to allocate a new
 1329  * one. Also, if we are passed an sa upcall, we need to dispatch
 1330  * it to the app.
 1331  */
 1332 static void
 1333 sa_switchcall(void *arg)
 1334 {
 1335         struct lwp *l, *l2;
 1336         struct proc *p;
 1337         struct sadata_vp *vp;
 1338         struct sadata_upcall *sau;
 1339         struct sastack *sast;
 1340         int s;
 1341 
 1342         l2 = curlwp;
 1343         p = l2->l_proc;
 1344         vp = l2->l_savp;
 1345         sau = arg;
 1346 
 1347         if (p->p_flag & P_WEXIT) {
 1348                 sadata_upcall_free(sau);
 1349                 lwp_exit(l2);
 1350         }
 1351 
 1352         KDASSERT(vp->savp_lwp == l2);
 1353         DPRINTFN(6,("sa_switchcall(%d.%d)\n", p->p_pid, l2->l_lid));
 1354 
 1355         l2->l_flag &= ~L_SA;
 1356         if (LIST_EMPTY(&vp->savp_lwpcache)) {
 1357                 /* Allocate the next cache LWP */
 1358                 DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n",
 1359                     p->p_pid, l2->l_lid));
 1360                 sa_newcachelwp(l2);
 1361         }
 1362         if (sau) {
 1363                 l = vp->savp_blocker;
 1364                 sast = sa_getstack(p->p_sa);
 1365                 if (sast) {
 1366                         sau->sau_stack = sast->sast_stack;
 1367                         SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
 1368                         l2->l_flag |= L_SA_UPCALL;
 1369                 } else {
 1370                         /*
 1371                          * Oops! We're in trouble. The app hasn't
 1372                          * passeed us in any stacks on which to deliver
 1373                          * the upcall.
 1374                          *
 1375                          * WRS: I think this code is wrong. If we can't
 1376                          * get a stack, we are dead. We either need
 1377                          * to block waiting for one (assuming there's a
 1378                          * live vp still in userland so it can hand back
 1379                          * stacks, or we should just kill the process
 1380                          * as we're deadlocked.
 1381                          */
 1382 #ifdef DIAGNOSTIC
 1383                         printf("sa_switchcall(%d.%d flag %x): Not enough stacks.\n",
 1384                             p->p_pid, l->l_lid, l->l_flag);
 1385 #endif
 1386                         if (vp->savp_sleeper_upcall == NULL)
 1387                                 vp->savp_sleeper_upcall = sau;
 1388                         else
 1389                                 sadata_upcall_free(sau);
 1390                         PHOLD(l2);
 1391                         SCHED_LOCK(s);
 1392                         sa_putcachelwp(p, l2); /* sets L_SA */
 1393                         vp->savp_lwp = l;
 1394                         l->l_flag &= ~L_SA_BLOCKING;
 1395                         p->p_nrlwps--;
 1396                         mi_switch(l2, NULL);
 1397                         /* mostly NOTREACHED */
 1398                         SCHED_ASSERT_UNLOCKED();
 1399                         splx(s);
 1400                 }
 1401         }
 1402         l2->l_flag |= L_SA;
 1403 
 1404         upcallret(l2);
 1405 }
 1406 
 1407 /*
 1408  * sa_newcachelwp
 1409  *      Allocate a new lwp, attach it to l's vp, and add it to
 1410  * the vp's idle cache.
 1411  *      Assumes no locks (other than kernel lock) on entry and exit.
 1412  * Locks scheduler lock during operation.
 1413  *      Returns 0 on success or if process is exiting. Returns ENOMEM
 1414  * if it is unable to allocate a new uarea.
 1415  */
 1416 static int
 1417 sa_newcachelwp(struct lwp *l)
 1418 {
 1419         struct proc *p;
 1420         struct lwp *l2;
 1421         vaddr_t uaddr;
 1422         boolean_t inmem;
 1423         int s;
 1424 
 1425         p = l->l_proc;
 1426         if (p->p_flag & P_WEXIT)
 1427                 return (0);
 1428 
 1429         inmem = uvm_uarea_alloc(&uaddr);
 1430         if (__predict_false(uaddr == 0)) {
 1431                 return (ENOMEM);
 1432         } else {
 1433                 newlwp(l, p, uaddr, inmem, 0, NULL, 0, child_return, 0, &l2);
 1434                 /* We don't want this LWP on the process's main LWP list, but
 1435                  * newlwp helpfully puts it there. Unclear if newlwp should
 1436                  * be tweaked.
 1437                  */
 1438                 PHOLD(l2);
 1439                 SCHED_LOCK(s);
 1440                 l2->l_savp = l->l_savp;
 1441                 sa_putcachelwp(p, l2);
 1442                 SCHED_UNLOCK(s);
 1443         }
 1444 
 1445         return (0);
 1446 }
 1447 
 1448 /*
 1449  * sa_putcachelwp
 1450  *      Take a normal process LWP and place it in the SA cache.
 1451  * LWP must not be running!
 1452  *      Scheduler lock held on entry and exit.
 1453  */
 1454 void
 1455 sa_putcachelwp(struct proc *p, struct lwp *l)
 1456 {
 1457         struct sadata_vp *vp;
 1458 
 1459         SCHED_ASSERT_LOCKED();
 1460 
 1461         vp = l->l_savp;
 1462 
 1463         LIST_REMOVE(l, l_sibling);
 1464         p->p_nlwps--;
 1465         l->l_stat = LSSUSPENDED;
 1466         l->l_flag |= (L_DETACHED | L_SA);
 1467         /* XXX lock sadata */
 1468         DPRINTFN(5,("sa_putcachelwp(%d.%d) Adding LWP %d to cache\n",
 1469             p->p_pid, curlwp->l_lid, l->l_lid));
 1470         LIST_INSERT_HEAD(&vp->savp_lwpcache, l, l_sibling);
 1471         vp->savp_ncached++;
 1472         /* XXX unlock */
 1473 }
 1474 
 1475 /*
 1476  * sa_getcachelwp
 1477  *      Fetch a LWP from the cache.
 1478  * Scheduler lock held on entry and exit.
 1479  */
 1480 struct lwp *
 1481 sa_getcachelwp(struct sadata_vp *vp)
 1482 {
 1483         struct lwp *l;
 1484         struct proc *p;
 1485 
 1486         SCHED_ASSERT_LOCKED();
 1487 
 1488         l = NULL;
 1489         /* XXX lock sadata */
 1490         if (vp->savp_ncached > 0) {
 1491                 vp->savp_ncached--;
 1492                 l = LIST_FIRST(&vp->savp_lwpcache);
 1493                 LIST_REMOVE(l, l_sibling);
 1494                 p = l->l_proc;
 1495                 LIST_INSERT_HEAD(&p->p_lwps, l, l_sibling);
 1496                 p->p_nlwps++;
 1497                 DPRINTFN(5,("sa_getcachelwp(%d.%d) Got LWP %d from cache.\n",
 1498                     p->p_pid, curlwp->l_lid, l->l_lid));
 1499         }
 1500         /* XXX unlock */
 1501         return l;
 1502 }
 1503 
 1504 
 1505 void
 1506 sa_unblock_userret(struct lwp *l)
 1507 {
 1508         struct proc *p;
 1509         struct lwp *l2;
 1510         struct sadata *sa;
 1511         struct sadata_vp *vp;
 1512         struct sadata_upcall *sau;
 1513         struct sastack *sast;
 1514         int f, s;
 1515 
 1516         p = l->l_proc;
 1517         sa = p->p_sa;
 1518         vp = l->l_savp;
 1519 
 1520         if (p->p_flag & P_WEXIT)
 1521                 return;
 1522 
 1523         SCHED_ASSERT_UNLOCKED();
 1524 
 1525         KERNEL_PROC_LOCK(l);
 1526         SA_LWP_STATE_LOCK(l, f);
 1527 
 1528         DPRINTFN(7,("sa_unblock_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
 1529             l->l_flag));
 1530 
 1531         sa_setwoken(l);
 1532         /* maybe NOTREACHED */
 1533 
 1534         SCHED_LOCK(s);
 1535         if (l != vp->savp_lwp) {
 1536                 /* Invoke an "unblocked" upcall */
 1537                 DPRINTFN(8,("sa_unblock_userret(%d.%d) unblocking\n",
 1538                     p->p_pid, l->l_lid));
 1539 
 1540                 l2 = sa_vp_repossess(l);
 1541 
 1542                 SCHED_UNLOCK(s);
 1543 
 1544                 if (l2 == NULL)
 1545                         lwp_exit(l);
 1546 
 1547                 sast = sa_getstack(sa);
 1548                 if (p->p_flag & P_WEXIT)
 1549                         lwp_exit(l);
 1550 
 1551                 sau = sadata_upcall_alloc(1);
 1552                 if (p->p_flag & P_WEXIT) {
 1553                         sadata_upcall_free(sau);
 1554                         lwp_exit(l);
 1555                 }
 1556 
 1557                 KDASSERT(l2 != NULL);
 1558                 PHOLD(l2);
 1559 
 1560                 KDASSERT(sast != NULL);
 1561                 DPRINTFN(9,("sa_unblock_userret(%d.%d) using stack %p\n",
 1562                     l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
 1563 
 1564                 /*
 1565                  * Defer saving the event lwp's state because a
 1566                  * PREEMPT upcall could be on the queue already.
 1567                  */
 1568                 sa_upcall0(sau, SA_UPCALL_UNBLOCKED | SA_UPCALL_DEFER_EVENT,
 1569                            l, l2, 0, NULL, NULL);
 1570                 sau->sau_stack = sast->sast_stack;
 1571 
 1572                 SCHED_LOCK(s);
 1573                 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
 1574                 l->l_flag |= L_SA_UPCALL;
 1575                 l->l_flag &= ~L_SA_BLOCKING;
 1576                 sa_putcachelwp(p, l2);
 1577         }
 1578         SCHED_UNLOCK(s);
 1579 
 1580         SA_LWP_STATE_UNLOCK(l, f);
 1581         KERNEL_PROC_UNLOCK(l);
 1582 }
 1583 
 1584 /*
 1585  * sa_upcall_userret
 1586  *      We are about to exit the kernel and return to userland, and
 1587  * userret() noticed we have upcalls pending. So deliver them.
 1588  *
 1589  *      This is the place where unblocking upcalls get generated. We
 1590  * allocate the stack & upcall event here. We may block doing so, but
 1591  * we lock our LWP state (clear L_SA for the moment) while doing so.
 1592  *
 1593  *      In the case of delivering multiple upcall events, we will end up
 1594  * writing multiple stacks out to userland at once. The last one we send
 1595  * out will be the first one run, then it will notice the others and
 1596  * run them.
 1597  *
 1598  * No locks held on entry or exit. We lock the scheduler during processing.
 1599  */
 1600 void
 1601 sa_upcall_userret(struct lwp *l)
 1602 {
 1603         struct lwp *l2;
 1604         struct proc *p;
 1605         struct sadata *sa;
 1606         struct sadata_vp *vp;
 1607         struct sadata_upcall *sau;
 1608         struct sastack *sast;
 1609         int f, s;
 1610 
 1611         p = l->l_proc;
 1612         sa = p->p_sa;
 1613         vp = l->l_savp;
 1614 
 1615         SCHED_ASSERT_UNLOCKED();
 1616 
 1617         KERNEL_PROC_LOCK(l);
 1618         SA_LWP_STATE_LOCK(l, f);
 1619 
 1620         DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
 1621             l->l_flag));
 1622 
 1623         KDASSERT((l->l_flag & L_SA_BLOCKING) == 0);
 1624 
 1625         sast = NULL;
 1626         if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL) {
 1627                 sast = sa_getstack(sa);
 1628                 if (sast == NULL) {
 1629                         SA_LWP_STATE_UNLOCK(l, f);
 1630                         KERNEL_PROC_UNLOCK(l);
 1631                         preempt(1);
 1632                         return;
 1633                 }
 1634         }
 1635         SCHED_LOCK(s);
 1636         if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL &&
 1637             sast != NULL) {
 1638                 /* Invoke an "unblocked" upcall */
 1639                 l2 = vp->savp_wokenq_head;
 1640                 vp->savp_wokenq_head = l2->l_forw;
 1641 
 1642                 DPRINTFN(9,("sa_upcall_userret(%d.%d) using stack %p\n",
 1643                     l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
 1644 
 1645                 SCHED_UNLOCK(s);
 1646 
 1647                 if (p->p_flag & P_WEXIT)
 1648                         lwp_exit(l);
 1649 
 1650                 DPRINTFN(8,("sa_upcall_userret(%d.%d) unblocking %d\n",
 1651                     p->p_pid, l->l_lid, l2->l_lid));
 1652 
 1653                 sau = sadata_upcall_alloc(1);
 1654                 if (p->p_flag & P_WEXIT) {
 1655                         sadata_upcall_free(sau);
 1656                         lwp_exit(l);
 1657                 }
 1658 
 1659                 sa_upcall0(sau, SA_UPCALL_UNBLOCKED, l2, l, 0, NULL, NULL);
 1660                 sau->sau_stack = sast->sast_stack;
 1661 
 1662                 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
 1663 
 1664                 l2->l_flag &= ~L_SA_BLOCKING;
 1665                 SCHED_LOCK(s);
 1666                 sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */
 1667                 SCHED_UNLOCK(s);
 1668         } else {
 1669                 SCHED_UNLOCK(s);
 1670                 if (sast)
 1671                         sa_setstackfree(sast, sa);
 1672         }
 1673 
 1674         KDASSERT(vp->savp_lwp == l);
 1675 
 1676         while (!SIMPLEQ_EMPTY(&vp->savp_upcalls))
 1677                 sa_makeupcalls(l);
 1678 
 1679         if (vp->savp_wokenq_head == NULL)
 1680                 l->l_flag &= ~L_SA_UPCALL;
 1681 
 1682         SA_LWP_STATE_UNLOCK(l, f);
 1683         KERNEL_PROC_UNLOCK(l);
 1684         return;
 1685 }
 1686 
 1687 #define SACOPYOUT(sae, type, kp, up) \
 1688         (((sae)->sae_sacopyout != NULL) ? \
 1689         (*(sae)->sae_sacopyout)((type), (kp), (void *)(up)) : \
 1690         copyout((kp), (void *)(up), sizeof(*(kp))))
 1691 
 1692 /*
 1693  * sa_makeupcalls
 1694  *      We're delivering the first upcall on lwp l, so
 1695  * copy everything out. We assigned the stack for this upcall
 1696  * when we enqueued it.
 1697  *
 1698  *      KERNEL_PROC_LOCK should be held on entry and exit, and
 1699  * SA_LWP_STATE should also be locked (L_SA temporarily disabled).
 1700  *
 1701  *      If the enqueued event was DEFERRED, this is the time when we set
 1702  * up the upcall event's state.
 1703  */
 1704 static inline void
 1705 sa_makeupcalls(struct lwp *l)
 1706 {
 1707         struct lwp *l2, *eventq;
 1708         struct proc *p;
 1709         const struct sa_emul *sae;
 1710         struct sadata *sa;
 1711         struct sadata_vp *vp;
 1712         uintptr_t sapp, sap;
 1713         struct sa_t self_sa;
 1714         struct sa_t *sas[3];
 1715 #ifdef KTRACE
 1716         struct sa_t **ksapp = NULL;
 1717 #endif
 1718         struct sadata_upcall *sau;
 1719         void *stack, *ap;
 1720         union sau_state *e_ss;
 1721         ucontext_t *kup, *up;
 1722         size_t sz, ucsize;
 1723         int i, nint, nevents, s, type, error;
 1724 
 1725         p = l->l_proc;
 1726         sae = p->p_emul->e_sa;
 1727         sa = p->p_sa;
 1728         vp = l->l_savp;
 1729         ucsize = sae->sae_ucsize;
 1730 
 1731         sau = SIMPLEQ_FIRST(&vp->savp_upcalls);
 1732         SIMPLEQ_REMOVE_HEAD(&vp->savp_upcalls, sau_next);
 1733 
 1734         if (sau->sau_flags & SAU_FLAG_DEFERRED_EVENT)
 1735                 sa_upcall_getstate(&sau->sau_event,
 1736                     sau->sau_event.ss_deferred.ss_lwp);
 1737         if (sau->sau_flags & SAU_FLAG_DEFERRED_INTERRUPTED)
 1738                 sa_upcall_getstate(&sau->sau_interrupted,
 1739                     sau->sau_interrupted.ss_deferred.ss_lwp);
 1740 
 1741 #ifdef __MACHINE_STACK_GROWS_UP
 1742         stack = sau->sau_stack.ss_sp;
 1743 #else
 1744         stack = (caddr_t)sau->sau_stack.ss_sp + sau->sau_stack.ss_size;
 1745 #endif
 1746         stack = STACK_ALIGN(stack, ALIGNBYTES);
 1747 
 1748         self_sa.sa_id = l->l_lid;
 1749         self_sa.sa_cpu = vp->savp_id;
 1750         sas[0] = &self_sa;
 1751         nevents = 0;
 1752         nint = 0;
 1753         if (sau->sau_event.ss_captured.ss_sa.sa_context != NULL) {
 1754                 if (copyout(&sau->sau_event.ss_captured.ss_ctx,
 1755                     sau->sau_event.ss_captured.ss_sa.sa_context,
 1756                     ucsize) != 0) {
 1757 #ifdef DIAGNOSTIC
 1758                         printf("sa_makeupcalls(%d.%d): couldn't copyout"
 1759                             " context of event LWP %d\n",
 1760                             p->p_pid, l->l_lid,
 1761                             sau->sau_event.ss_captured.ss_sa.sa_id);
 1762 #endif
 1763                         sigexit(l, SIGILL);
 1764                         /* NOTREACHED */
 1765                 }
 1766                 sas[1] = &sau->sau_event.ss_captured.ss_sa;
 1767                 nevents = 1;
 1768         }
 1769         if (sau->sau_interrupted.ss_captured.ss_sa.sa_context != NULL) {
 1770                 KDASSERT(sau->sau_interrupted.ss_captured.ss_sa.sa_context !=
 1771                     sau->sau_event.ss_captured.ss_sa.sa_context);
 1772                 if (copyout(&sau->sau_interrupted.ss_captured.ss_ctx,
 1773                     sau->sau_interrupted.ss_captured.ss_sa.sa_context,
 1774                     ucsize) != 0) {
 1775 #ifdef DIAGNOSTIC
 1776                         printf("sa_makeupcalls(%d.%d): couldn't copyout"
 1777                             " context of interrupted LWP %d\n",
 1778                             p->p_pid, l->l_lid,
 1779                             sau->sau_interrupted.ss_captured.ss_sa.sa_id);
 1780 #endif
 1781                         sigexit(l, SIGILL);
 1782                         /* NOTREACHED */
 1783                 }
 1784                 sas[2] = &sau->sau_interrupted.ss_captured.ss_sa;
 1785                 nint = 1;
 1786         }
 1787         eventq = NULL;
 1788         if (sau->sau_type == SA_UPCALL_UNBLOCKED) {
 1789                 SCHED_LOCK(s);
 1790                 eventq = vp->savp_wokenq_head;
 1791                 vp->savp_wokenq_head = NULL;
 1792                 SCHED_UNLOCK(s);
 1793                 l2 = eventq;
 1794                 while (l2 != NULL) {
 1795                         nevents++;
 1796                         l2 = l2->l_forw;
 1797                 }
 1798         }
 1799 
 1800         /* Copy out the activation's ucontext */
 1801         up = (void *)STACK_ALLOC(stack, ucsize);
 1802         stack = STACK_GROW(stack, ucsize);
 1803         kup = kmem_zalloc(sizeof(*kup), KM_SLEEP);
 1804         kup->uc_stack = sau->sau_stack;
 1805         kup->uc_flags = _UC_STACK;
 1806         error = SACOPYOUT(sae, SAOUT_UCONTEXT, kup, up);
 1807         kmem_free(kup, sizeof(*kup));
 1808         if (error) {
 1809                 sadata_upcall_free(sau);
 1810 #ifdef DIAGNOSTIC
 1811                 printf("sa_makeupcalls: couldn't copyout activation"
 1812                     " ucontext for %d.%d to %p\n", l->l_proc->p_pid, l->l_lid,
 1813                     up);
 1814 #endif
 1815                 sigexit(l, SIGILL);
 1816                 /* NOTREACHED */
 1817         }
 1818         sas[0]->sa_context = up;
 1819 
 1820         /* Next, copy out the sa_t's and pointers to them. */
 1821 
 1822         sz = (1 + nevents + nint) * sae->sae_sasize;
 1823         sap = (uintptr_t)STACK_ALLOC(stack, sz);
 1824         sap += sz;
 1825         stack = STACK_GROW(stack, sz);
 1826 
 1827         sz = (1 + nevents + nint) * sae->sae_sapsize;
 1828         sapp = (uintptr_t)STACK_ALLOC(stack, sz);
 1829         sapp += sz;
 1830         stack = STACK_GROW(stack, sz);
 1831 
 1832 #ifdef KTRACE
 1833         if (KTRPOINT(p, KTR_SAUPCALL))
 1834                 ksapp = kmem_alloc(sizeof(struct sa_t *) * (nevents + nint + 1),
 1835                     KM_SLEEP);
 1836 #endif
 1837         KDASSERT(nint <= 1);
 1838         e_ss = NULL;
 1839         for (i = nevents + nint; i >= 0; i--) {
 1840                 struct sa_t *sasp;
 1841 
 1842                 sap -= sae->sae_sasize;
 1843                 sapp -= sae->sae_sapsize;
 1844                 error = 0;
 1845                 if (i == 1 + nevents)   /* interrupted sa */
 1846                         sasp = sas[2];
 1847                 else if (i <= 1)        /* self_sa and event sa */
 1848                         sasp = sas[i];
 1849                 else {                  /* extra sas */
 1850                         KDASSERT(sau->sau_type == SA_UPCALL_UNBLOCKED);
 1851                         KDASSERT(eventq != NULL);
 1852                         l2 = eventq;
 1853                         KDASSERT(l2 != NULL);
 1854                         eventq = l2->l_forw;
 1855                         DPRINTFN(8,
 1856                             ("sa_makeupcalls(%d.%d) unblocking extra %d\n",
 1857                             p->p_pid, l->l_lid, l2->l_lid));
 1858                         if (e_ss == NULL) {
 1859                                 e_ss = kmem_alloc(sizeof(*e_ss), KM_SLEEP);
 1860                         }
 1861                         sa_upcall_getstate(e_ss, l2);
 1862                         SCHED_LOCK(s);
 1863                         l2->l_flag &= ~L_SA_BLOCKING;
 1864                         sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */
 1865                         SCHED_UNLOCK(s);
 1866 
 1867                         error = copyout(&e_ss->ss_captured.ss_ctx,
 1868                             e_ss->ss_captured.ss_sa.sa_context, ucsize);
 1869                         sasp = &e_ss->ss_captured.ss_sa;
 1870                 }
 1871                 if (error != 0 ||
 1872                     SACOPYOUT(sae, SAOUT_SA_T, sasp, sap) ||
 1873                     SACOPYOUT(sae, SAOUT_SAP_T, &sap, sapp)) {
 1874                         /* Copying onto the stack didn't work. Die. */
 1875                         sadata_upcall_free(sau);
 1876 #ifdef DIAGNOSTIC
 1877                         printf("sa_makeupcalls(%d.%d): couldn't copyout\n",
 1878                             p->p_pid, l->l_lid);
 1879 #endif
 1880                         if (e_ss != NULL) {
 1881                                 kmem_free(e_ss, sizeof(*e_ss));
 1882                         }
 1883                         goto fail;
 1884                 }
 1885 #ifdef KTRACE
 1886                 if (KTRPOINT(p, KTR_SAUPCALL))
 1887                         ksapp[i] = sasp;
 1888 #endif
 1889 
 1890         }
 1891         if (e_ss != NULL) {
 1892                 kmem_free(e_ss, sizeof(*e_ss));
 1893         }
 1894         KDASSERT(eventq == NULL);
 1895 
 1896         /* Copy out the arg, if any */
 1897         /* xxx assume alignment works out; everything so far has been
 1898          * a structure, so...
 1899          */
 1900         if (sau->sau_arg) {
 1901                 ap = STACK_ALLOC(stack, sau->sau_argsize);
 1902                 stack = STACK_GROW(stack, sau->sau_argsize);
 1903                 if (copyout(sau->sau_arg, ap, sau->sau_argsize) != 0) {
 1904                         /* Copying onto the stack didn't work. Die. */
 1905                         sadata_upcall_free(sau);
 1906 #ifdef DIAGNOSTIC
 1907                         printf("sa_makeupcalls(%d.%d): couldn't copyout"
 1908                             " sadata_upcall arg %p size %ld to %p \n",
 1909                             p->p_pid, l->l_lid,
 1910                             sau->sau_arg, (long) sau->sau_argsize, ap);
 1911 #endif
 1912                         goto fail;
 1913                 }
 1914         } else {
 1915                 ap = NULL;
 1916 #ifdef __hppa__
 1917                 stack = STACK_ALIGN(stack, HPPA_FRAME_SIZE);
 1918 #endif
 1919         }
 1920         type = sau->sau_type;
 1921 
 1922         if (vp->savp_sleeper_upcall == NULL)
 1923                 vp->savp_sleeper_upcall = sau;
 1924         else
 1925                 sadata_upcall_free(sau);
 1926 
 1927         DPRINTFN(7,("sa_makeupcalls(%d.%d): type %d\n", p->p_pid,
 1928             l->l_lid, type));
 1929 
 1930 #ifdef KTRACE
 1931         if (KTRPOINT(p, KTR_SAUPCALL)) {
 1932                 ktrsaupcall(l, type, nevents, nint, (void *)sapp, ap, ksapp);
 1933                 kmem_free(ksapp, sizeof(struct sa_t *) * (nevents + nint + 1));
 1934         }
 1935 #endif
 1936         (*sae->sae_upcall)(l, type, nevents, nint, (void *)sapp, ap, stack,
 1937             sa->sa_upcall);
 1938 
 1939         l->l_flag &= ~L_SA_YIELD;
 1940         return;
 1941 
 1942 fail:
 1943 #ifdef KTRACE
 1944         if (KTRPOINT(p, KTR_SAUPCALL))
 1945                 kmem_free(ksapp, sizeof(struct sa_t) * (nevents + nint + 1));
 1946 #endif
 1947         sigexit(l, SIGILL);
 1948         /* NOTREACHED */
 1949 }
 1950 
 1951 static void
 1952 sa_setwoken(struct lwp *l)
 1953 {
 1954         struct lwp *l2, *vp_lwp;
 1955         struct proc *p;
 1956         struct sadata *sa;
 1957         struct sadata_vp *vp;
 1958         int s;
 1959 
 1960         SCHED_LOCK(s);
 1961 
 1962         if ((l->l_flag & L_SA_BLOCKING) == 0) {
 1963                 SCHED_UNLOCK(s);
 1964                 return;
 1965         }
 1966 
 1967         p = l->l_proc;
 1968         sa = p->p_sa;
 1969         vp = l->l_savp;
 1970         vp_lwp = vp->savp_lwp;
 1971         l2 = NULL;
 1972 
 1973         KDASSERT(vp_lwp != NULL);
 1974         DPRINTFN(3,("sa_setwoken(%d.%d) woken, flags %x, vp %d\n",
 1975                      l->l_proc->p_pid, l->l_lid, l->l_flag,
 1976                      vp_lwp->l_lid));
 1977 
 1978 #if notyet
 1979         if (vp_lwp->l_flag & L_SA_IDLE) {
 1980                 KDASSERT((vp_lwp->l_flag & L_SA_UPCALL) == 0);
 1981                 KDASSERT(vp->savp_wokenq_head == NULL);
 1982                 DPRINTFN(3,
 1983                     ("sa_setwoken(%d.%d) repossess: idle vp_lwp %d state %d\n",
 1984                     l->l_proc->p_pid, l->l_lid,
 1985                     vp_lwp->l_lid, vp_lwp->l_stat));
 1986                 vp_lwp->l_flag &= ~L_SA_IDLE;
 1987                 SCHED_UNLOCK(s);
 1988                 return;
 1989         }
 1990 #endif
 1991 
 1992         DPRINTFN(3,("sa_setwoken(%d.%d) put on wokenq: vp_lwp %d state %d\n",
 1993                      l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid,
 1994                      vp_lwp->l_stat));
 1995 
 1996         PHOLD(l);
 1997         if (vp->savp_wokenq_head == NULL)
 1998                 vp->savp_wokenq_head = l;
 1999         else
 2000                 *vp->savp_wokenq_tailp = l;
 2001         *(vp->savp_wokenq_tailp = &l->l_forw) = NULL;
 2002 
 2003         switch (vp_lwp->l_stat) {
 2004         case LSONPROC:
 2005                 if (vp_lwp->l_flag & L_SA_UPCALL)
 2006                         break;
 2007                 vp_lwp->l_flag |= L_SA_UPCALL;
 2008                 if (vp_lwp->l_flag & L_SA_YIELD)
 2009                         break;
 2010                 /* XXX IPI vp_lwp->l_cpu */
 2011                 break;
 2012         case LSSLEEP:
 2013                 if (vp_lwp->l_flag & L_SA_IDLE) {
 2014                         vp_lwp->l_flag &= ~L_SA_IDLE;
 2015                         vp_lwp->l_flag |= L_SA_UPCALL;
 2016                         setrunnable(vp_lwp);
 2017                         break;
 2018                 }
 2019                 vp_lwp->l_flag |= L_SA_UPCALL;
 2020                 break;
 2021         case LSSUSPENDED:
 2022 #ifdef DIAGNOSTIC
 2023                 printf("sa_setwoken(%d.%d) vp lwp %d LSSUSPENDED\n",
 2024                     l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid);
 2025 #endif
 2026                 break;
 2027         case LSSTOP:
 2028                 vp_lwp->l_flag |= L_SA_UPCALL;
 2029                 break;
 2030         case LSRUN:
 2031                 if (vp_lwp->l_flag & L_SA_UPCALL)
 2032                         break;
 2033                 vp_lwp->l_flag |= L_SA_UPCALL;
 2034                 if (vp_lwp->l_flag & L_SA_YIELD)
 2035                         break;
 2036                 if (vp_lwp->l_slptime > 1) {
 2037                         void updatepri(struct lwp *);
 2038                         updatepri(vp_lwp);
 2039                 }
 2040                 vp_lwp->l_slptime = 0;
 2041                 if (vp_lwp->l_flag & L_INMEM) {
 2042                         if (vp_lwp->l_cpu == curcpu())
 2043                                 l2 = vp_lwp;
 2044                         else
 2045                                 need_resched(vp_lwp->l_cpu);
 2046                 } else
 2047                         sched_wakeup(&proc0);
 2048                 break;
 2049         default:
 2050                 panic("sa_vp LWP not sleeping/onproc/runnable");
 2051         }
 2052 
 2053         l->l_stat = LSSUSPENDED;
 2054         p->p_nrlwps--;
 2055         mi_switch(l, l2);
 2056         /* maybe NOTREACHED */
 2057         SCHED_ASSERT_UNLOCKED();
 2058         splx(s);
 2059         if (p->p_flag & P_WEXIT)
 2060                 lwp_exit(l);
 2061 }
 2062 
 2063 static struct lwp *
 2064 sa_vp_repossess(struct lwp *l)
 2065 {
 2066         struct lwp *l2;
 2067         struct proc *p = l->l_proc;
 2068         struct sadata_vp *vp = l->l_savp;
 2069 
 2070         SCHED_ASSERT_LOCKED();
 2071 
 2072         /*
 2073          * Put ourselves on the virtual processor and note that the
 2074          * previous occupant of that position was interrupted.
 2075          */
 2076         l2 = vp->savp_lwp;
 2077         vp->savp_lwp = l;
 2078         if (l2) {
 2079                 if (l2->l_flag & L_SA_YIELD)
 2080                         l2->l_flag &= ~(L_SA_YIELD|L_SA_IDLE);
 2081 
 2082                 DPRINTFN(1,("sa_vp_repossess(%d.%d) vp lwp %d state %d\n",
 2083                              p->p_pid, l->l_lid, l2->l_lid, l2->l_stat));
 2084 
 2085                 KDASSERT(l2 != l);
 2086                 switch (l2->l_stat) {
 2087                 case LSRUN:
 2088                         remrunqueue(l2);
 2089                         p->p_nrlwps--;
 2090                         break;
 2091                 case LSSLEEP:
 2092                         unsleep(l2);
 2093                         l2->l_flag &= ~L_SINTR;
 2094                         break;
 2095                 case LSSUSPENDED:
 2096 #ifdef DIAGNOSTIC
 2097                         printf("sa_vp_repossess(%d.%d) vp lwp %d LSSUSPENDED\n",
 2098                             l->l_proc->p_pid, l->l_lid, l2->l_lid);
 2099 #endif
 2100                         break;
 2101 #ifdef DIAGNOSTIC
 2102                 default:
 2103                         panic("SA VP %d.%d is in state %d, not running"
 2104                             " or sleeping\n", p->p_pid, l2->l_lid,
 2105                             l2->l_stat);
 2106 #endif
 2107                 }
 2108                 l2->l_stat = LSSUSPENDED;
 2109         }
 2110         return l2;
 2111 }
 2112 
 2113 
 2114 
 2115 #ifdef DEBUG
 2116 int debug_print_sa(struct proc *);
 2117 int debug_print_lwp(struct lwp *);
 2118 int debug_print_proc(int);
 2119 
 2120 int
 2121 debug_print_proc(int pid)
 2122 {
 2123         struct proc *p;
 2124 
 2125         p = pfind(pid);
 2126         if (p == NULL)
 2127                 printf("No process %d\n", pid);
 2128         else
 2129                 debug_print_sa(p);
 2130 
 2131         return 0;
 2132 }
 2133 
 2134 int
 2135 debug_print_sa(struct proc *p)
 2136 {
 2137         struct lwp *l;
 2138         struct sadata *sa;
 2139         struct sadata_vp *vp;
 2140 
 2141         printf("Process %d (%s), state %d, address %p, flags %x\n",
 2142             p->p_pid, p->p_comm, p->p_stat, p, p->p_flag);
 2143         printf("LWPs: %d (%d running, %d zombies)\n",
 2144             p->p_nlwps, p->p_nrlwps, p->p_nzlwps);
 2145         LIST_FOREACH(l, &p->p_lwps, l_sibling)
 2146             debug_print_lwp(l);
 2147         sa = p->p_sa;
 2148         if (sa) {
 2149                 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) {
 2150                         if (vp->savp_lwp)
 2151                                 printf("SA VP: %d %s\n", vp->savp_lwp->l_lid,
 2152                                     vp->savp_lwp->l_flag & L_SA_YIELD ?
 2153                                     (vp->savp_lwp->l_flag & L_SA_IDLE ?
 2154                                         "idle" : "yielding") : "");
 2155                         printf("SAs: %d cached LWPs\n", vp->savp_ncached);
 2156                         LIST_FOREACH(l, &vp->savp_lwpcache, l_sibling)
 2157                                 debug_print_lwp(l);
 2158                 }
 2159         }
 2160 
 2161         return 0;
 2162 }
 2163 
 2164 int
 2165 debug_print_lwp(struct lwp *l)
 2166 {
 2167 
 2168         printf("LWP %d address %p ", l->l_lid, l);
 2169         printf("state %d flags %x ", l->l_stat, l->l_flag);
 2170         if (l->l_wchan)
 2171                 printf("wait %p %s", l->l_wchan, l->l_wmesg);
 2172         printf("\n");
 2173 
 2174         return 0;
 2175 }
 2176 
 2177 #endif

Cache object: 0a3ed08972cdfb6baca2688ae27c9594


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.