FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_sa.c
1 /* $NetBSD: kern_sa.c,v 1.87.2.2 2008/09/16 18:49:34 bouyer Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2004, 2005 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the NetBSD
21 * Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38
39 #include <sys/cdefs.h>
40
41 #include "opt_ktrace.h"
42 #include "opt_multiprocessor.h"
43 __KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.87.2.2 2008/09/16 18:49:34 bouyer Exp $");
44
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/pool.h>
48 #include <sys/proc.h>
49 #include <sys/types.h>
50 #include <sys/ucontext.h>
51 #include <sys/kmem.h>
52 #include <sys/mount.h>
53 #include <sys/sa.h>
54 #include <sys/savar.h>
55 #include <sys/syscallargs.h>
56 #include <sys/ktrace.h>
57
58 #include <uvm/uvm_extern.h>
59
60 static POOL_INIT(sadata_pool, sizeof(struct sadata), 0, 0, 0, "sadatapl",
61 &pool_allocator_nointr); /* memory pool for sadata structures */
62 static POOL_INIT(saupcall_pool, sizeof(struct sadata_upcall), 0, 0, 0,
63 "saupcpl", &pool_allocator_nointr); /* memory pool for pending upcalls */
64 static POOL_INIT(sastack_pool, sizeof(struct sastack), 0, 0, 0, "sastackpl",
65 &pool_allocator_nointr); /* memory pool for sastack structs */
66 static POOL_INIT(savp_pool, sizeof(struct sadata_vp), 0, 0, 0, "savppl",
67 &pool_allocator_nointr); /* memory pool for sadata_vp structures */
68
69 static struct sadata_vp *sa_newsavp(struct sadata *);
70 static inline int sa_stackused(struct sastack *, struct sadata *);
71 static inline void sa_setstackfree(struct sastack *, struct sadata *);
72 static struct sastack *sa_getstack(struct sadata *);
73 static inline struct sastack *sa_getstack0(struct sadata *);
74 static inline int sast_compare(struct sastack *, struct sastack *);
75 #ifdef MULTIPROCESSOR
76 static int sa_increaseconcurrency(struct lwp *, int);
77 #endif
78 static void sa_setwoken(struct lwp *);
79 static void sa_switchcall(void *);
80 static int sa_newcachelwp(struct lwp *);
81 static inline void sa_makeupcalls(struct lwp *);
82 static struct lwp *sa_vp_repossess(struct lwp *l);
83
84 static inline int sa_pagefault(struct lwp *, ucontext_t *);
85
86 static void sa_upcall0(struct sadata_upcall *, int, struct lwp *, struct lwp *,
87 size_t, void *, void (*)(void *));
88 static void sa_upcall_getstate(union sau_state *, struct lwp *);
89
90 #define SA_DEBUG
91
92 #ifdef SA_DEBUG
93 #define DPRINTF(x) do { if (sadebug) printf_nolog x; } while (0)
94 #define DPRINTFN(n,x) do { if (sadebug & (1<<(n-1))) printf_nolog x; } while (0)
95 int sadebug = 0;
96 #else
97 #define DPRINTF(x)
98 #define DPRINTFN(n,x)
99 #endif
100
101
102 #define SA_LWP_STATE_LOCK(l, f) do { \
103 (f) = (l)->l_flag; \
104 (l)->l_flag &= ~L_SA; \
105 } while (/*CONSTCOND*/ 0)
106
107 #define SA_LWP_STATE_UNLOCK(l, f) do { \
108 (l)->l_flag |= (f) & L_SA; \
109 } while (/*CONSTCOND*/ 0)
110
111 SPLAY_PROTOTYPE(sasttree, sastack, sast_node, sast_compare);
112 SPLAY_GENERATE(sasttree, sastack, sast_node, sast_compare);
113
114 /*
115 * sa_critpath API
116 * permit other parts of the kernel to make SA_LWP_STATE_{UN,}LOCK calls.
117 */
118 void
119 sa_critpath_enter(struct lwp *l1, sa_critpath_t *f1)
120 {
121 SA_LWP_STATE_LOCK(l1, *f1);
122 }
123 void
124 sa_critpath_exit(struct lwp *l1, sa_critpath_t *f1)
125 {
126 SA_LWP_STATE_UNLOCK(l1, *f1);
127 }
128
129
130 /*
131 * sadata_upcall_alloc:
132 *
133 * Allocate an sadata_upcall structure.
134 */
135 struct sadata_upcall *
136 sadata_upcall_alloc(int waitok)
137 {
138 struct sadata_upcall *sau;
139
140 sau = pool_get(&saupcall_pool, waitok ? PR_WAITOK : PR_NOWAIT);
141 if (sau) {
142 sau->sau_arg = NULL;
143 }
144 return sau;
145 }
146
147 /*
148 * sadata_upcall_free:
149 *
150 * Free an sadata_upcall structure and any associated argument data.
151 */
152 void
153 sadata_upcall_free(struct sadata_upcall *sau)
154 {
155
156 if (sau == NULL) {
157 return;
158 }
159 if (sau->sau_arg) {
160 (*sau->sau_argfreefunc)(sau->sau_arg);
161 }
162 pool_put(&saupcall_pool, sau);
163 }
164
165 /*
166 * sa_newsavp
167 *
168 * Allocate a new virtual processor structure, do some simple
169 * initialization and add it to the passed-in sa. Pre-allocate
170 * an upcall event data structure for when the main thread on
171 * this vp blocks.
172 *
173 * We lock sa_lock while manipulating the list of vp's.
174 *
175 * We allocate the lwp to run on this separately. In the case of the
176 * first lwp/vp for a process, the lwp already exists. It's the
177 * main (only) lwp of the process.
178 */
179 static struct sadata_vp *
180 sa_newsavp(struct sadata *sa)
181 {
182 struct sadata_vp *vp, *qvp;
183 struct sadata_upcall *sau;
184
185 /* Allocate virtual processor data structure */
186 vp = pool_get(&savp_pool, PR_WAITOK);
187 /* And preallocate an upcall data structure for sleeping */
188 sau = sadata_upcall_alloc(1);
189 /* Initialize. */
190 memset(vp, 0, sizeof(*vp));
191 simple_lock_init(&vp->savp_lock);
192 vp->savp_lwp = NULL;
193 vp->savp_wokenq_head = NULL;
194 vp->savp_faultaddr = 0;
195 vp->savp_ofaultaddr = 0;
196 LIST_INIT(&vp->savp_lwpcache);
197 vp->savp_ncached = 0;
198 vp->savp_sleeper_upcall = sau;
199 SIMPLEQ_INIT(&vp->savp_upcalls);
200
201 simple_lock(&sa->sa_lock);
202 /* find first free savp_id and add vp to sorted slist */
203 if (SLIST_EMPTY(&sa->sa_vps) ||
204 SLIST_FIRST(&sa->sa_vps)->savp_id != 0) {
205 vp->savp_id = 0;
206 SLIST_INSERT_HEAD(&sa->sa_vps, vp, savp_next);
207 } else {
208 SLIST_FOREACH(qvp, &sa->sa_vps, savp_next) {
209 if (SLIST_NEXT(qvp, savp_next) == NULL ||
210 SLIST_NEXT(qvp, savp_next)->savp_id !=
211 qvp->savp_id + 1)
212 break;
213 }
214 vp->savp_id = qvp->savp_id + 1;
215 SLIST_INSERT_AFTER(qvp, vp, savp_next);
216 }
217 simple_unlock(&sa->sa_lock);
218
219 return (vp);
220 }
221
222 /*
223 * sys_sa_register
224 * Handle copyin and copyout of info for registering the
225 * upcall handler address.
226 */
227 int
228 sys_sa_register(struct lwp *l, void *v, register_t *retval)
229 {
230 struct sys_sa_register_args /* {
231 syscallarg(sa_upcall_t) new;
232 syscallarg(sa_upcall_t *) old;
233 syscallarg(int) flags;
234 syscallarg(ssize_t) stackinfo_offset;
235 } */ *uap = v;
236 int error;
237 sa_upcall_t prev;
238
239 error = dosa_register(l, SCARG(uap, new), &prev, SCARG(uap, flags),
240 SCARG(uap, stackinfo_offset));
241 if (error)
242 return error;
243
244 if (SCARG(uap, old))
245 return copyout(&prev, SCARG(uap, old),
246 sizeof(prev));
247 return 0;
248 }
249
250 /*
251 * dosa_register
252 *
253 * Change the upcall address for the process. If needed, allocate
254 * an sadata structure (and initialize it) for the process. If initializing,
255 * set the flags in the sadata structure to those passed in. Flags will
256 * be ignored if the sadata structure already exists (dosa_regiister was
257 * already called).
258 *
259 * Note: changing the upcall handler address for a process that has
260 * concurrency greater than one can yield ambiguous results. The one
261 * guarantee we can offer is that any upcalls generated on all CPUs
262 * after this routine finishes will use the new upcall handler. Note
263 * that any upcalls delivered upon return to user level by the
264 * sys_sa_register() system call that called this routine will use the
265 * new upcall handler. Note that any such upcalls will be delivered
266 * before the old upcall handling address has been returned to
267 * the application.
268 */
269 int
270 dosa_register(struct lwp *l, sa_upcall_t new, sa_upcall_t *prev, int flags,
271 ssize_t stackinfo_offset)
272 {
273 struct proc *p = l->l_proc;
274 struct sadata *sa;
275
276 if (p->p_sa == NULL) {
277 /* Allocate scheduler activations data structure */
278 sa = pool_get(&sadata_pool, PR_WAITOK);
279 /* Initialize. */
280 memset(sa, 0, sizeof(*sa));
281 simple_lock_init(&sa->sa_lock);
282 sa->sa_flag = flags & SA_FLAG_ALL;
283 sa->sa_maxconcurrency = 1;
284 sa->sa_concurrency = 1;
285 SPLAY_INIT(&sa->sa_stackstree);
286 sa->sa_stacknext = NULL;
287 if (flags & SA_FLAG_STACKINFO)
288 sa->sa_stackinfo_offset = stackinfo_offset;
289 else
290 sa->sa_stackinfo_offset = 0;
291 sa->sa_nstacks = 0;
292 SLIST_INIT(&sa->sa_vps);
293 p->p_sa = sa;
294 KASSERT(l->l_savp == NULL);
295 }
296 if (l->l_savp == NULL) {
297 l->l_savp = sa_newsavp(p->p_sa);
298 sa_newcachelwp(l);
299 }
300
301 *prev = p->p_sa->sa_upcall;
302 p->p_sa->sa_upcall = new;
303
304 return (0);
305 }
306
307 void
308 sa_release(struct proc *p)
309 {
310 struct sadata *sa;
311 struct sastack *sast, *next;
312 struct sadata_vp *vp;
313 struct lwp *l;
314
315 sa = p->p_sa;
316 KDASSERT(sa != NULL);
317 KASSERT(p->p_nlwps <= 1);
318
319 for (sast = SPLAY_MIN(sasttree, &sa->sa_stackstree); sast != NULL;
320 sast = next) {
321 next = SPLAY_NEXT(sasttree, &sa->sa_stackstree, sast);
322 SPLAY_REMOVE(sasttree, &sa->sa_stackstree, sast);
323 pool_put(&sastack_pool, sast);
324 }
325
326 p->p_flag &= ~P_SA;
327 while ((vp = SLIST_FIRST(&p->p_sa->sa_vps)) != NULL) {
328 SLIST_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next);
329 if (vp->savp_sleeper_upcall) {
330 sadata_upcall_free(vp->savp_sleeper_upcall);
331 vp->savp_sleeper_upcall = NULL;
332 }
333 pool_put(&savp_pool, vp);
334 }
335 pool_put(&sadata_pool, sa);
336 p->p_sa = NULL;
337 l = LIST_FIRST(&p->p_lwps);
338 if (l) {
339 KASSERT(LIST_NEXT(l, l_sibling) == NULL);
340 l->l_savp = NULL;
341 }
342 }
343
344 /*
345 * sa_fetchstackgen
346 *
347 * copyin the generation number for the stack in question.
348 *
349 * WRS: I think this routine needs the SA_LWP_STATE_LOCK() dance, either
350 * here or in its caller.
351 */
352 static int
353 sa_fetchstackgen(struct sastack *sast, struct sadata *sa, unsigned int *gen)
354 {
355 int error;
356
357 /* COMPAT_NETBSD32: believe it or not, but the following is ok */
358 error = copyin(&((struct sa_stackinfo_t *)
359 ((char *)sast->sast_stack.ss_sp +
360 sa->sa_stackinfo_offset))->sasi_stackgen, gen, sizeof(*gen));
361
362 return error;
363 }
364
365 /*
366 * sa_stackused
367 *
368 * Convenience routine to determine if a given stack has been used
369 * or not. We consider a stack to be unused if the kernel's concept
370 * of its generation number matches that of userland.
371 * We kill the application with SIGILL if there is an error copying
372 * in the userland generation number.
373 */
374 static inline int
375 sa_stackused(struct sastack *sast, struct sadata *sa)
376 {
377 unsigned int gen;
378
379 if (sa_fetchstackgen(sast, sa, &gen)) {
380 #ifdef DIAGNOSTIC
381 printf("sa_stackused: couldn't copyin sasi_stackgen");
382 #endif
383 sigexit(curlwp, SIGILL);
384 /* NOTREACHED */
385 }
386 return (sast->sast_gen != gen);
387 }
388
389 /*
390 * sa_setstackfree
391 *
392 * Convenience routine to mark a stack as unused in the kernel's
393 * eyes. We do this by setting the kernel's generation number for the stack
394 * to that of userland.
395 * We kill the application with SIGILL if there is an error copying
396 * in the userland generation number.
397 */
398 static inline void
399 sa_setstackfree(struct sastack *sast, struct sadata *sa)
400 {
401 unsigned int gen;
402
403 if (sa_fetchstackgen(sast, sa, &gen)) {
404 #ifdef DIAGNOSTIC
405 printf("sa_setstackfree: couldn't copyin sasi_stackgen");
406 #endif
407 sigexit(curlwp, SIGILL);
408 /* NOTREACHED */
409 }
410 sast->sast_gen = gen;
411 }
412
413 /*
414 * sa_getstack
415 *
416 * Find next free stack, starting at sa->sa_stacknext.
417 *
418 * Caller must have the splay tree locked and should have cleared L_SA for
419 * our thread. This is not the time to go generating upcalls as we aren't
420 * in a position to deliver another one.
421 */
422 static struct sastack *
423 sa_getstack(struct sadata *sa)
424 {
425 struct sastack *sast;
426
427 SCHED_ASSERT_UNLOCKED();
428
429 if ((sast = sa->sa_stacknext) == NULL || sa_stackused(sast, sa))
430 sast = sa_getstack0(sa);
431
432 if (sast == NULL)
433 return NULL;
434
435 sast->sast_gen++;
436
437 return sast;
438 }
439
440 /*
441 * sa_getstack0 -- get the lowest numbered sa stack
442 *
443 * We walk the splay tree in order and find the lowest-numbered
444 * (as defined by SPLAY_MIN() and SPLAY_NEXT() ordering) stack that
445 * is unused.
446 */
447 static inline struct sastack *
448 sa_getstack0(struct sadata *sa)
449 {
450 struct sastack *start;
451
452 if (sa->sa_stacknext == NULL) {
453 sa->sa_stacknext = SPLAY_MIN(sasttree, &sa->sa_stackstree);
454 if (sa->sa_stacknext == NULL)
455 return NULL;
456 }
457 start = sa->sa_stacknext;
458
459 while (sa_stackused(sa->sa_stacknext, sa)) {
460 sa->sa_stacknext = SPLAY_NEXT(sasttree, &sa->sa_stackstree,
461 sa->sa_stacknext);
462 if (sa->sa_stacknext == NULL)
463 sa->sa_stacknext = SPLAY_MIN(sasttree,
464 &sa->sa_stackstree);
465 if (sa->sa_stacknext == start)
466 return NULL;
467 }
468 return sa->sa_stacknext;
469 }
470
471 /*
472 * sast_compare - compare two sastacks
473 *
474 * We sort stacks according to their userspace addresses.
475 * Stacks are "equal" if their start + size overlap.
476 */
477 static inline int
478 sast_compare(struct sastack *a, struct sastack *b)
479 {
480 if ((vaddr_t)a->sast_stack.ss_sp + a->sast_stack.ss_size <=
481 (vaddr_t)b->sast_stack.ss_sp)
482 return (-1);
483 if ((vaddr_t)a->sast_stack.ss_sp >=
484 (vaddr_t)b->sast_stack.ss_sp + b->sast_stack.ss_size)
485 return (1);
486 return (0);
487 }
488
489 /*
490 * sa_copyin_stack -- copyin a stack.
491 */
492 static int
493 sa_copyin_stack(stack_t *stacks, int index, stack_t *dest)
494 {
495 return copyin(stacks + index, dest, sizeof(stack_t));
496 }
497
498 /*
499 * sys_sa_stacks -- the user level threading library is passing us stacks
500 *
501 * We copy in some arguments then call sa_stacks1() to do the main
502 * work. NETBSD32 has its own front-end for this call.
503 */
504 int
505 sys_sa_stacks(struct lwp *l, void *v, register_t *retval)
506 {
507 struct sys_sa_stacks_args /* {
508 syscallarg(int) num;
509 syscallarg(stack_t *) stacks;
510 } */ *uap = v;
511
512 return sa_stacks1(l, retval, SCARG(uap, num), SCARG(uap, stacks), sa_copyin_stack);
513 }
514
515 /*
516 * sa_stacks1
517 * Process stacks passed-in by the user threading library. At
518 * present we use the kernel lock to lock the SPLAY tree, which we
519 * manipulate to load in the stacks.
520 *
521 * It is an error to pass in a stack that we already know about
522 * and which hasn't been used. Passing in a known-but-used one is fine.
523 * We accept up to SA_MAXNUMSTACKS per desired vp (concurrency level).
524 */
525 int
526 sa_stacks1(struct lwp *l, register_t *retval, int num, stack_t *stacks,
527 sa_copyin_stack_t do_sa_copyin_stack)
528 {
529 struct sadata *sa = l->l_proc->p_sa;
530 struct sastack *sast, newsast;
531 int count, error, f, i;
532
533 /* We have to be using scheduler activations */
534 if (sa == NULL)
535 return (EINVAL);
536
537 count = num;
538 if (count < 0)
539 return (EINVAL);
540
541 SA_LWP_STATE_LOCK(l, f);
542
543 error = 0;
544
545 for (i = 0; i < count; i++) {
546 error = do_sa_copyin_stack(stacks, i, &newsast.sast_stack);
547 if (error) {
548 count = i;
549 break;
550 }
551 sast = SPLAY_FIND(sasttree, &sa->sa_stackstree, &newsast);
552 if (sast != NULL) {
553 DPRINTFN(9, ("sa_stacks(%d.%d) returning stack %p\n",
554 l->l_proc->p_pid, l->l_lid,
555 newsast.sast_stack.ss_sp));
556 if (sa_stackused(sast, sa) == 0) {
557 count = i;
558 error = EEXIST;
559 break;
560 }
561 } else if (sa->sa_nstacks >=
562 SA_MAXNUMSTACKS * sa->sa_concurrency) {
563 DPRINTFN(9,
564 ("sa_stacks(%d.%d) already using %d stacks\n",
565 l->l_proc->p_pid, l->l_lid,
566 SA_MAXNUMSTACKS * sa->sa_concurrency));
567 count = i;
568 error = ENOMEM;
569 break;
570 } else {
571 DPRINTFN(9, ("sa_stacks(%d.%d) adding stack %p\n",
572 l->l_proc->p_pid, l->l_lid,
573 newsast.sast_stack.ss_sp));
574 sast = pool_get(&sastack_pool, PR_WAITOK);
575 sast->sast_stack = newsast.sast_stack;
576 SPLAY_INSERT(sasttree, &sa->sa_stackstree, sast);
577 sa->sa_nstacks++;
578 }
579 sa_setstackfree(sast, sa);
580 }
581
582 SA_LWP_STATE_UNLOCK(l, f);
583
584 *retval = count;
585 return (error);
586 }
587
588
589 /*
590 * sys_sa_enable - throw the switch & enable SA
591 *
592 * Fairly simple. Make sure the sadata and vp've been set up for this
593 * process, assign this thread to the vp and initiate the first upcall
594 * (SA_UPCALL_NEWPROC).
595 */
596 int
597 sys_sa_enable(struct lwp *l, void *v, register_t *retval)
598 {
599 struct proc *p = l->l_proc;
600 struct sadata *sa = p->p_sa;
601 struct sadata_vp *vp = l->l_savp;
602 int error;
603
604 DPRINTF(("sys_sa_enable(%d.%d)\n", l->l_proc->p_pid,
605 l->l_lid));
606
607 /* We have to be using scheduler activations */
608 if (sa == NULL || vp == NULL)
609 return (EINVAL);
610
611 if (p->p_flag & P_SA) /* Already running! */
612 return (EBUSY);
613
614 error = sa_upcall(l, SA_UPCALL_NEWPROC, l, NULL, 0, NULL, NULL);
615 if (error)
616 return (error);
617
618 /* Assign this LWP to the virtual processor */
619 vp->savp_lwp = l;
620
621 p->p_flag |= P_SA;
622 l->l_flag |= L_SA; /* We are now an activation LWP */
623
624 /*
625 * This will return to the SA handler previously registered.
626 */
627 return (0);
628 }
629
630
631 /*
632 * sa_increaseconcurrency
633 * Raise the process's maximum concurrency level to the
634 * requested level. Does nothing if the current maximum councurrency
635 * is greater than the requested.
636 * Uses the kernel lock to implicitly lock operations. Also
637 * uses sa_lock to lock the vp list and uses SCHED_LOCK() to lock the
638 * scheduler.
639 */
640 #ifdef MULTIPROCESSOR
641 static int
642 sa_increaseconcurrency(struct lwp *l, int concurrency)
643 {
644 struct proc *p;
645 struct lwp *l2;
646 struct sadata *sa;
647 vaddr_t uaddr;
648 boolean_t inmem;
649 int addedconcurrency, error, s;
650 struct sadata_vp *vp;
651
652 p = l->l_proc;
653 sa = p->p_sa;
654
655 addedconcurrency = 0;
656 simple_lock(&sa->sa_lock);
657 while (sa->sa_maxconcurrency < concurrency) {
658 sa->sa_maxconcurrency++;
659 sa->sa_concurrency++;
660 simple_unlock(&sa->sa_lock);
661
662 inmem = uvm_uarea_alloc(&uaddr);
663 if (__predict_false(uaddr == 0)) {
664 /* reset concurrency */
665 simple_lock(&sa->sa_lock);
666 sa->sa_maxconcurrency--;
667 sa->sa_concurrency--;
668 simple_unlock(&sa->sa_lock);
669 return (addedconcurrency);
670 } else {
671 newlwp(l, p, uaddr, inmem, 0, NULL, 0,
672 child_return, 0, &l2);
673 l2->l_flag |= L_SA;
674 l2->l_savp = vp = sa_newsavp(sa);
675 if (vp) {
676 vp->savp_lwp = l2;
677 cpu_setfunc(l2, sa_switchcall, NULL);
678 error = sa_upcall(l2, SA_UPCALL_NEWPROC,
679 NULL, NULL, 0, NULL, NULL);
680 if (error) {
681 /* free new savp */
682 SLIST_REMOVE(&sa->sa_vps, vp,
683 sadata_vp, savp_next);
684 if (vp->savp_sleeper_upcall) {
685 sadata_upcall_free(
686 vp->savp_sleeper_upcall);
687 vp->savp_sleeper_upcall = NULL;
688 }
689 pool_put(&savp_pool, vp);
690 }
691 } else
692 error = 1;
693 if (error) {
694 /* put l2 into l's VP LWP cache */
695 l2->l_savp = l->l_savp;
696 PHOLD(l2);
697 SCHED_LOCK(s);
698 sa_putcachelwp(p, l2);
699 SCHED_UNLOCK(s);
700 /* reset concurrency */
701 simple_lock(&sa->sa_lock);
702 sa->sa_maxconcurrency--;
703 sa->sa_concurrency--;
704 simple_unlock(&sa->sa_lock);
705 return (addedconcurrency);
706 }
707 SCHED_LOCK(s);
708 setrunnable(l2);
709 SCHED_UNLOCK(s);
710 addedconcurrency++;
711 }
712 simple_lock(&sa->sa_lock);
713 }
714 simple_unlock(&sa->sa_lock);
715
716 return (addedconcurrency);
717 }
718 #endif
719
720 /*
721 * sys_sa_setconcurrency
722 * The user threading library wants to increase the number
723 * of active virtual CPUS we assign to it. We return the number of virt
724 * CPUs we assigned to the process. We limit concurrency to the number
725 * of CPUs in the system.
726 *
727 * WRS: at present, this system call serves two purposes. The first is
728 * for an application to indicate that it wants a certain concurrency
729 * level. The second is for the application to request that the kernel
730 * reactivate previously allocated virtual CPUs.
731 */
732 int
733 sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval)
734 {
735 struct sys_sa_setconcurrency_args /* {
736 syscallarg(int) concurrency;
737 } */ *uap = v;
738 struct sadata *sa = l->l_proc->p_sa;
739 #ifdef MULTIPROCESSOR
740 struct sadata_vp *vp = l->l_savp;
741 int ncpus, s;
742 struct cpu_info *ci;
743 CPU_INFO_ITERATOR cii;
744 #endif
745
746 DPRINTFN(11,("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid,
747 l->l_lid));
748
749 /* We have to be using scheduler activations */
750 if (sa == NULL)
751 return (EINVAL);
752
753 if ((l->l_proc->p_flag & P_SA) == 0)
754 return (EINVAL);
755
756 if (SCARG(uap, concurrency) < 1)
757 return (EINVAL);
758
759 *retval = 0;
760 /*
761 * Concurrency greater than the number of physical CPUs does
762 * not make sense.
763 * XXX Should we ever support hot-plug CPUs, this will need
764 * adjustment.
765 */
766 #ifdef MULTIPROCESSOR
767 if (SCARG(uap, concurrency) > sa->sa_maxconcurrency) {
768 ncpus = 0;
769 for (CPU_INFO_FOREACH(cii, ci))
770 ncpus++;
771 *retval += sa_increaseconcurrency(l,
772 min(SCARG(uap, concurrency), ncpus));
773 }
774 #endif
775
776 DPRINTFN(11,("sys_sa_concurrency(%d.%d) want %d, have %d, max %d\n",
777 l->l_proc->p_pid, l->l_lid, SCARG(uap, concurrency),
778 sa->sa_concurrency, sa->sa_maxconcurrency));
779 #ifdef MULTIPROCESSOR
780 if (SCARG(uap, concurrency) > sa->sa_concurrency) {
781 SCHED_LOCK(s);
782 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) {
783 if (vp->savp_lwp->l_flag & L_SA_IDLE) {
784 vp->savp_lwp->l_flag &=
785 ~(L_SA_IDLE|L_SA_YIELD|L_SINTR);
786 SCHED_UNLOCK(s);
787 DPRINTFN(11,("sys_sa_concurrency(%d.%d) "
788 "NEWPROC vp %d\n",
789 l->l_proc->p_pid, l->l_lid,
790 vp->savp_id));
791 cpu_setfunc(vp->savp_lwp, sa_switchcall, NULL);
792 /* error = */ sa_upcall(vp->savp_lwp,
793 SA_UPCALL_NEWPROC,
794 NULL, NULL, 0, NULL, NULL);
795 SCHED_LOCK(s);
796 sa->sa_concurrency++;
797 setrunnable(vp->savp_lwp);
798 KDASSERT((vp->savp_lwp->l_flag & L_SINTR) == 0);
799 (*retval)++;
800 }
801 if (sa->sa_concurrency == SCARG(uap, concurrency))
802 break;
803 }
804 SCHED_UNLOCK(s);
805 }
806 #endif
807
808 return (0);
809 }
810
811 /*
812 * sys_sa_yield
813 * application has nothing for this lwp to do, so let it linger in
814 * the kernel.
815 */
816 int
817 sys_sa_yield(struct lwp *l, void *v, register_t *retval)
818 {
819 struct proc *p = l->l_proc;
820
821 if (p->p_sa == NULL || !(p->p_flag & P_SA)) {
822 DPRINTFN(1,
823 ("sys_sa_yield(%d.%d) proc %p not SA (p_sa %p, flag %s)\n",
824 p->p_pid, l->l_lid, p, p->p_sa,
825 p->p_flag & P_SA ? "T" : "F"));
826 return (EINVAL);
827 }
828
829 sa_yield(l);
830
831 return (EJUSTRETURN);
832 }
833
834 /*
835 * sa_yield
836 * This lwp has nothing to do, so hang around. Assuming we
837 * are the lwp "on" our vp, tsleep in "sawait" until there's something
838 * to do. If there are upcalls, we deliver them explicitly.
839 */
840 void
841 sa_yield(struct lwp *l)
842 {
843 struct proc *p = l->l_proc;
844 struct sadata *sa = p->p_sa;
845 struct sadata_vp *vp = l->l_savp;
846 int ret;
847
848 KERNEL_LOCK_ASSERT_LOCKED();
849
850 if (vp->savp_lwp != l) {
851 /*
852 * We lost the VP on our way here, this happens for
853 * instance when we sleep in systrace. This will end
854 * in an SA_UNBLOCKED_UPCALL in sa_setwoken().
855 */
856 DPRINTFN(1,("sa_yield(%d.%d) lost VP\n",
857 p->p_pid, l->l_lid));
858 KDASSERT(l->l_flag & L_SA_BLOCKING);
859 return;
860 }
861
862 /*
863 * If we're the last running LWP, stick around to receive
864 * signals.
865 */
866 KDASSERT((l->l_flag & L_SA_YIELD) == 0);
867 DPRINTFN(1,("sa_yield(%d.%d) going dormant\n",
868 p->p_pid, l->l_lid));
869 /*
870 * A signal will probably wake us up. Worst case, the upcall
871 * happens and just causes the process to yield again.
872 */
873 /* s = splsched(); */ /* Protect from timer expirations */
874 KDASSERT(vp->savp_lwp == l);
875 /*
876 * If we were told to make an upcall or exit before
877 * the splsched(), make sure we process it instead of
878 * going to sleep. It might make more sense for this to
879 * be handled inside of tsleep....
880 */
881 ret = 0;
882 l->l_flag |= L_SA_YIELD;
883 if (l->l_flag & L_SA_UPCALL) {
884 /* KERNEL_PROC_UNLOCK(l); in upcallret() */
885 upcallret(l);
886 KERNEL_PROC_LOCK(l);
887 }
888 while (l->l_flag & L_SA_YIELD) {
889 DPRINTFN(1,("sa_yield(%d.%d) really going dormant\n",
890 p->p_pid, l->l_lid));
891
892 simple_lock(&sa->sa_lock);
893 sa->sa_concurrency--;
894 simple_unlock(&sa->sa_lock);
895
896 ret = tsleep(l, PUSER | PCATCH, "sawait", 0);
897
898 simple_lock(&sa->sa_lock);
899 sa->sa_concurrency++;
900 simple_unlock(&sa->sa_lock);
901
902 KDASSERT(vp->savp_lwp == l || p->p_flag & P_WEXIT);
903
904 /* KERNEL_PROC_UNLOCK(l); in upcallret() */
905 upcallret(l);
906 KERNEL_PROC_LOCK(l);
907 }
908 /* splx(s); */
909 DPRINTFN(1,("sa_yield(%d.%d) returned, ret %d, userret %p\n",
910 p->p_pid, l->l_lid, ret, p->p_userret));
911 }
912
913
914 /*
915 * sys_sa_preempt - preempt a running thread
916 *
917 * Given an lwp id, send it a user upcall. This is a way for libpthread to
918 * kick something into the upcall handler.
919 */
920 int
921 sys_sa_preempt(struct lwp *l, void *v, register_t *retval)
922 {
923 struct sys_sa_preempt_args /* {
924 syscallarg(int) sa_id;
925 } */ *uap = v;
926 struct sadata *sa = l->l_proc->p_sa;
927 struct lwp *t;
928 int target, s, error;
929
930 DPRINTFN(11,("sys_sa_preempt(%d.%d)\n", l->l_proc->p_pid,
931 l->l_lid));
932
933 /* We have to be using scheduler activations */
934 if (sa == NULL)
935 return (EINVAL);
936
937 if ((l->l_proc->p_flag & P_SA) == 0)
938 return (EINVAL);
939
940 if ((target = SCARG(uap, sa_id)) < 1)
941 return (EINVAL);
942
943 SCHED_LOCK(s);
944
945 LIST_FOREACH(t, &l->l_proc->p_lwps, l_sibling)
946 if (t->l_lid == target)
947 break;
948
949 if (t == NULL) {
950 error = ESRCH;
951 goto exit_lock;
952 }
953
954 /* XXX WRS We really need all of this locking documented */
955 SCHED_UNLOCK(s);
956
957 error = sa_upcall(l, SA_UPCALL_USER | SA_UPCALL_DEFER_EVENT, l, NULL,
958 0, NULL, NULL);
959 if (error)
960 return error;
961
962 return 0;
963
964 exit_lock:
965 SCHED_UNLOCK(s);
966
967 return error;
968 }
969
970
971 /* XXX Hm, naming collision. */
972 void
973 sa_preempt(struct lwp *l)
974 {
975 struct proc *p = l->l_proc;
976 struct sadata *sa = p->p_sa;
977
978 /*
979 * Defer saving the lwp's state because on some ports
980 * preemption can occur between generating an unblocked upcall
981 * and processing the upcall queue.
982 */
983 if (sa->sa_flag & SA_FLAG_PREEMPT)
984 sa_upcall(l, SA_UPCALL_PREEMPTED | SA_UPCALL_DEFER_EVENT,
985 l, NULL, 0, NULL, NULL);
986 }
987
988
989 /*
990 * Set up the user-level stack and trapframe to do an upcall.
991 *
992 * NOTE: This routine WILL FREE "arg" in the case of failure! Callers
993 * should not touch the "arg" pointer once calling sa_upcall().
994 */
995 int
996 sa_upcall(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted,
997 size_t argsize, void *arg, void (*func)(void *))
998 {
999 struct sadata_upcall *sau;
1000 struct sadata *sa = l->l_proc->p_sa;
1001 struct sadata_vp *vp = l->l_savp;
1002 struct sastack *sast;
1003 int f, error;
1004
1005 /* XXX prevent recursive upcalls if we sleep for memory */
1006 SA_LWP_STATE_LOCK(l, f);
1007 sast = sa_getstack(sa);
1008 SA_LWP_STATE_UNLOCK(l, f);
1009 if (sast == NULL) {
1010 return (ENOMEM);
1011 }
1012 DPRINTFN(9,("sa_upcall(%d.%d) using stack %p\n",
1013 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
1014
1015 if (l->l_proc->p_emul->e_sa->sae_upcallconv) {
1016 error = (*l->l_proc->p_emul->e_sa->sae_upcallconv)(l, type,
1017 &argsize, &arg, &func);
1018 if (error)
1019 return error;
1020 }
1021
1022 SA_LWP_STATE_LOCK(l, f);
1023 sau = sadata_upcall_alloc(1);
1024 SA_LWP_STATE_UNLOCK(l, f);
1025 sa_upcall0(sau, type, event, interrupted, argsize, arg, func);
1026 sau->sau_stack = sast->sast_stack;
1027
1028 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
1029 l->l_flag |= L_SA_UPCALL;
1030
1031 return (0);
1032 }
1033
1034 static void
1035 sa_upcall0(struct sadata_upcall *sau, int type, struct lwp *event,
1036 struct lwp *interrupted, size_t argsize, void *arg, void (*func)(void *))
1037 {
1038
1039 KDASSERT((event == NULL) || (event != interrupted));
1040
1041 sau->sau_flags = 0;
1042
1043 if (type & SA_UPCALL_DEFER_EVENT) {
1044 sau->sau_event.ss_deferred.ss_lwp = event;
1045 sau->sau_flags |= SAU_FLAG_DEFERRED_EVENT;
1046 } else
1047 sa_upcall_getstate(&sau->sau_event, event);
1048 if (type & SA_UPCALL_DEFER_INTERRUPTED) {
1049 sau->sau_interrupted.ss_deferred.ss_lwp = interrupted;
1050 sau->sau_flags |= SAU_FLAG_DEFERRED_INTERRUPTED;
1051 } else
1052 sa_upcall_getstate(&sau->sau_interrupted, interrupted);
1053
1054 sau->sau_type = type & SA_UPCALL_TYPE_MASK;
1055 sau->sau_argsize = argsize;
1056 sau->sau_arg = arg;
1057 sau->sau_argfreefunc = func;
1058 }
1059
1060 /*
1061 * sa_ucsp
1062 * return the stack pointer (??) for a given context as
1063 * reported by the _UC_MACHINE_SP() macro.
1064 */
1065 void *
1066 sa_ucsp(void *arg)
1067 {
1068 ucontext_t *uc = arg;
1069
1070 return (void *)(uintptr_t)_UC_MACHINE_SP(uc);
1071 }
1072
1073 /*
1074 * sa_upcall_getstate
1075 * Fill in the given sau_state with info for the passed-in
1076 * lwp, and update the lwp accordingly.
1077 * WRS: unwaware of any locking before entry!
1078 * We set L_SA_SWITCHING on the target lwp. We assume that l_flag is
1079 * protected by the kernel lock and untouched in interrupt context.
1080 */
1081 static void
1082 sa_upcall_getstate(union sau_state *ss, struct lwp *l)
1083 {
1084 caddr_t sp;
1085 size_t ucsize;
1086
1087 if (l) {
1088 l->l_flag |= L_SA_SWITCHING;
1089 (*l->l_proc->p_emul->e_sa->sae_getucontext)(l,
1090 (void *)&ss->ss_captured.ss_ctx);
1091 l->l_flag &= ~L_SA_SWITCHING;
1092 sp = (*l->l_proc->p_emul->e_sa->sae_ucsp)
1093 (&ss->ss_captured.ss_ctx);
1094 /* XXX COMPAT_NETBSD32: _UC_UCONTEXT_ALIGN */
1095 sp = STACK_ALIGN(sp, ~_UC_UCONTEXT_ALIGN);
1096 ucsize = roundup(l->l_proc->p_emul->e_sa->sae_ucsize,
1097 (~_UC_UCONTEXT_ALIGN) + 1);
1098 ss->ss_captured.ss_sa.sa_context =
1099 (ucontext_t *)STACK_ALLOC(sp, ucsize);
1100 ss->ss_captured.ss_sa.sa_id = l->l_lid;
1101 ss->ss_captured.ss_sa.sa_cpu = l->l_savp->savp_id;
1102 } else
1103 ss->ss_captured.ss_sa.sa_context = NULL;
1104 }
1105
1106
1107 /*
1108 * sa_pagefault
1109 *
1110 * Detect double pagefaults and pagefaults on upcalls.
1111 * - double pagefaults are detected by comparing the previous faultaddr
1112 * against the current faultaddr
1113 * - pagefaults on upcalls are detected by checking if the userspace
1114 * thread is running on an upcall stack
1115 */
1116 static inline int
1117 sa_pagefault(struct lwp *l, ucontext_t *l_ctx)
1118 {
1119 struct proc *p;
1120 struct sadata *sa;
1121 struct sadata_vp *vp;
1122 struct sastack sast;
1123
1124 p = l->l_proc;
1125 sa = p->p_sa;
1126 vp = l->l_savp;
1127
1128 KDASSERT(vp->savp_lwp == l);
1129
1130 if (vp->savp_faultaddr == vp->savp_ofaultaddr) {
1131 DPRINTFN(10,("sa_pagefault(%d.%d) double page fault\n",
1132 p->p_pid, l->l_lid));
1133 return 1;
1134 }
1135
1136 sast.sast_stack.ss_sp = (*p->p_emul->e_sa->sae_ucsp)(l_ctx);
1137 sast.sast_stack.ss_size = 1;
1138
1139 if (SPLAY_FIND(sasttree, &sa->sa_stackstree, &sast)) {
1140 DPRINTFN(10,("sa_pagefault(%d.%d) upcall page fault\n",
1141 p->p_pid, l->l_lid));
1142 return 1;
1143 }
1144
1145 vp->savp_ofaultaddr = vp->savp_faultaddr;
1146 return 0;
1147 }
1148
1149
1150 /*
1151 * sa_switch
1152 *
1153 * Called by tsleep() when it wants to call mi_switch().
1154 * Block current LWP and switch to another.
1155 *
1156 * WE ARE NOT ALLOWED TO SLEEP HERE! WE ARE CALLED FROM WITHIN
1157 * TSLEEP() ITSELF! We are called with sched_lock held, and must
1158 * hold it right through the mi_switch() call.
1159 *
1160 * We return with the scheduler unlocked.
1161 *
1162 * We are called in one of three conditions:
1163 *
1164 * 1: We are an sa_yield thread. If there are any UNBLOCKED
1165 * upcalls to deliver, deliver them (by exiting) instead of sleeping.
1166 * 2: We are the main lwp (we're the lwp on our vp). Trigger
1167 * delivery of a BLOCKED upcall.
1168 * 3: We are not the main lwp on our vp. Chances are we got
1169 * woken up but the sleeper turned around and went back to sleep.
1170 * It seems that select and poll do this a lot. So just go back to sleep.
1171 */
1172
1173 void
1174 sa_switch(struct lwp *l, int type)
1175 {
1176 struct proc *p = l->l_proc;
1177 struct sadata_vp *vp = l->l_savp;
1178 struct sadata_upcall *sau = NULL;
1179 struct lwp *l2;
1180 int s;
1181
1182 DPRINTFN(4,("sa_switch(%d.%d type %d VP %d)\n", p->p_pid, l->l_lid,
1183 type, vp->savp_lwp ? vp->savp_lwp->l_lid : 0));
1184
1185 SCHED_ASSERT_LOCKED();
1186
1187 if (p->p_flag & P_WEXIT) {
1188 mi_switch(l, NULL);
1189 return;
1190 }
1191
1192 if (l->l_flag & L_SA_YIELD) {
1193
1194 /*
1195 * Case 0: we're blocking in sa_yield
1196 */
1197 if (vp->savp_wokenq_head == NULL && p->p_userret == NULL) {
1198 l->l_flag |= L_SA_IDLE;
1199 mi_switch(l, NULL);
1200 } else {
1201 /* make us running again. */
1202 unsleep(l);
1203 l->l_stat = LSONPROC;
1204 l->l_proc->p_nrlwps++;
1205 s = splsched();
1206 SCHED_UNLOCK(s);
1207 }
1208 return;
1209 } else if (vp->savp_lwp == l) {
1210 /*
1211 * Case 1: we're blocking for the first time; generate
1212 * a SA_BLOCKED upcall and allocate resources for the
1213 * UNBLOCKED upcall.
1214 */
1215 if (vp->savp_sleeper_upcall) {
1216 sau = vp->savp_sleeper_upcall;
1217 vp->savp_sleeper_upcall = NULL;
1218 }
1219
1220 if (sau == NULL) {
1221 #ifdef DIAGNOSTIC
1222 printf("sa_switch(%d.%d): no upcall data.\n",
1223 p->p_pid, l->l_lid);
1224 #endif
1225 panic("Oops! Don't have a sleeper!\n");
1226 /* XXXWRS Shouldn't we just kill the app here? */
1227 mi_switch(l, NULL);
1228 return;
1229 }
1230
1231 /*
1232 * The process of allocating a new LWP could cause
1233 * sleeps. We're called from inside sleep, so that
1234 * would be Bad. Therefore, we must use a cached new
1235 * LWP. The first thing that this new LWP must do is
1236 * allocate another LWP for the cache. */
1237 l2 = sa_getcachelwp(vp);
1238 if (l2 == NULL) {
1239 /* XXXSMP */
1240 /* No upcall for you! */
1241 /* XXX The consequences of this are more subtle and
1242 * XXX the recovery from this situation deserves
1243 * XXX more thought.
1244 */
1245
1246 /* XXXUPSXXX Should only happen with concurrency > 1 */
1247 #ifdef DIAGNOSTIC
1248 printf("sa_switch(%d.%d): no cached LWP for upcall.\n",
1249 p->p_pid, l->l_lid);
1250 #endif
1251 mi_switch(l, NULL);
1252 sadata_upcall_free(sau);
1253 return;
1254 }
1255
1256 cpu_setfunc(l2, sa_switchcall, sau);
1257 sa_upcall0(sau, SA_UPCALL_BLOCKED, l, NULL, 0, NULL, NULL);
1258
1259 /*
1260 * Perform the double/upcall pagefault check.
1261 * We do this only here since we need l's ucontext to
1262 * get l's userspace stack. sa_upcall0 above has saved
1263 * it for us.
1264 * The L_SA_PAGEFAULT flag is set in the MD
1265 * pagefault code to indicate a pagefault. The MD
1266 * pagefault code also saves the faultaddr for us.
1267 */
1268 if ((l->l_flag & L_SA_PAGEFAULT) && sa_pagefault(l,
1269 &sau->sau_event.ss_captured.ss_ctx) != 0) {
1270 cpu_setfunc(l2, sa_switchcall, NULL);
1271 sa_putcachelwp(p, l2); /* PHOLD from sa_getcachelwp */
1272 mi_switch(l, NULL);
1273 /*
1274 * WRS Not sure how vp->savp_sleeper_upcall != NULL
1275 * but be careful none the less
1276 */
1277 if (vp->savp_sleeper_upcall == NULL)
1278 vp->savp_sleeper_upcall = sau;
1279 else
1280 sadata_upcall_free(sau);
1281 DPRINTFN(10,("sa_switch(%d.%d) page fault resolved\n",
1282 p->p_pid, l->l_lid));
1283 if (vp->savp_faultaddr == vp->savp_ofaultaddr)
1284 vp->savp_ofaultaddr = -1;
1285 return;
1286 }
1287
1288 DPRINTFN(8,("sa_switch(%d.%d) blocked upcall %d\n",
1289 p->p_pid, l->l_lid, l2->l_lid));
1290
1291 l->l_flag |= L_SA_BLOCKING;
1292 l2->l_priority = l2->l_usrpri;
1293 vp->savp_blocker = l;
1294 vp->savp_lwp = l2;
1295 setrunnable(l2);
1296 PRELE(l2); /* Remove the artificial hold-count */
1297
1298 KDASSERT(l2 != l);
1299 } else if (vp->savp_lwp != NULL) {
1300
1301 /*
1302 * Case 2: We've been woken up while another LWP was
1303 * on the VP, but we're going back to sleep without
1304 * having returned to userland and delivering the
1305 * SA_UNBLOCKED upcall (select and poll cause this
1306 * kind of behavior a lot).
1307 */
1308 l2 = NULL;
1309 } else {
1310 /* NOTREACHED */
1311 panic("sa_vp empty");
1312 }
1313
1314 DPRINTFN(4,("sa_switch(%d.%d) switching to LWP %d.\n",
1315 p->p_pid, l->l_lid, l2 ? l2->l_lid : 0));
1316 mi_switch(l, l2);
1317 DPRINTFN(4,("sa_switch(%d.%d flag %x) returned.\n",
1318 p->p_pid, l->l_lid, l->l_flag));
1319 KDASSERT(l->l_wchan == 0);
1320
1321 SCHED_ASSERT_UNLOCKED();
1322 }
1323
1324 /*
1325 * sa_switchcall
1326 *
1327 * We need to pass an upcall to userland. We are now
1328 * running on a spare stack and need to allocate a new
1329 * one. Also, if we are passed an sa upcall, we need to dispatch
1330 * it to the app.
1331 */
1332 static void
1333 sa_switchcall(void *arg)
1334 {
1335 struct lwp *l, *l2;
1336 struct proc *p;
1337 struct sadata_vp *vp;
1338 struct sadata_upcall *sau;
1339 struct sastack *sast;
1340 int s;
1341
1342 l2 = curlwp;
1343 p = l2->l_proc;
1344 vp = l2->l_savp;
1345 sau = arg;
1346
1347 if (p->p_flag & P_WEXIT) {
1348 sadata_upcall_free(sau);
1349 lwp_exit(l2);
1350 }
1351
1352 KDASSERT(vp->savp_lwp == l2);
1353 DPRINTFN(6,("sa_switchcall(%d.%d)\n", p->p_pid, l2->l_lid));
1354
1355 l2->l_flag &= ~L_SA;
1356 if (LIST_EMPTY(&vp->savp_lwpcache)) {
1357 /* Allocate the next cache LWP */
1358 DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n",
1359 p->p_pid, l2->l_lid));
1360 sa_newcachelwp(l2);
1361 }
1362 if (sau) {
1363 l = vp->savp_blocker;
1364 sast = sa_getstack(p->p_sa);
1365 if (sast) {
1366 sau->sau_stack = sast->sast_stack;
1367 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
1368 l2->l_flag |= L_SA_UPCALL;
1369 } else {
1370 /*
1371 * Oops! We're in trouble. The app hasn't
1372 * passeed us in any stacks on which to deliver
1373 * the upcall.
1374 *
1375 * WRS: I think this code is wrong. If we can't
1376 * get a stack, we are dead. We either need
1377 * to block waiting for one (assuming there's a
1378 * live vp still in userland so it can hand back
1379 * stacks, or we should just kill the process
1380 * as we're deadlocked.
1381 */
1382 #ifdef DIAGNOSTIC
1383 printf("sa_switchcall(%d.%d flag %x): Not enough stacks.\n",
1384 p->p_pid, l->l_lid, l->l_flag);
1385 #endif
1386 if (vp->savp_sleeper_upcall == NULL)
1387 vp->savp_sleeper_upcall = sau;
1388 else
1389 sadata_upcall_free(sau);
1390 PHOLD(l2);
1391 SCHED_LOCK(s);
1392 sa_putcachelwp(p, l2); /* sets L_SA */
1393 vp->savp_lwp = l;
1394 l->l_flag &= ~L_SA_BLOCKING;
1395 p->p_nrlwps--;
1396 mi_switch(l2, NULL);
1397 /* mostly NOTREACHED */
1398 SCHED_ASSERT_UNLOCKED();
1399 splx(s);
1400 }
1401 }
1402 l2->l_flag |= L_SA;
1403
1404 upcallret(l2);
1405 }
1406
1407 /*
1408 * sa_newcachelwp
1409 * Allocate a new lwp, attach it to l's vp, and add it to
1410 * the vp's idle cache.
1411 * Assumes no locks (other than kernel lock) on entry and exit.
1412 * Locks scheduler lock during operation.
1413 * Returns 0 on success or if process is exiting. Returns ENOMEM
1414 * if it is unable to allocate a new uarea.
1415 */
1416 static int
1417 sa_newcachelwp(struct lwp *l)
1418 {
1419 struct proc *p;
1420 struct lwp *l2;
1421 vaddr_t uaddr;
1422 boolean_t inmem;
1423 int s;
1424
1425 p = l->l_proc;
1426 if (p->p_flag & P_WEXIT)
1427 return (0);
1428
1429 inmem = uvm_uarea_alloc(&uaddr);
1430 if (__predict_false(uaddr == 0)) {
1431 return (ENOMEM);
1432 } else {
1433 newlwp(l, p, uaddr, inmem, 0, NULL, 0, child_return, 0, &l2);
1434 /* We don't want this LWP on the process's main LWP list, but
1435 * newlwp helpfully puts it there. Unclear if newlwp should
1436 * be tweaked.
1437 */
1438 PHOLD(l2);
1439 SCHED_LOCK(s);
1440 l2->l_savp = l->l_savp;
1441 sa_putcachelwp(p, l2);
1442 SCHED_UNLOCK(s);
1443 }
1444
1445 return (0);
1446 }
1447
1448 /*
1449 * sa_putcachelwp
1450 * Take a normal process LWP and place it in the SA cache.
1451 * LWP must not be running!
1452 * Scheduler lock held on entry and exit.
1453 */
1454 void
1455 sa_putcachelwp(struct proc *p, struct lwp *l)
1456 {
1457 struct sadata_vp *vp;
1458
1459 SCHED_ASSERT_LOCKED();
1460
1461 vp = l->l_savp;
1462
1463 LIST_REMOVE(l, l_sibling);
1464 p->p_nlwps--;
1465 l->l_stat = LSSUSPENDED;
1466 l->l_flag |= (L_DETACHED | L_SA);
1467 /* XXX lock sadata */
1468 DPRINTFN(5,("sa_putcachelwp(%d.%d) Adding LWP %d to cache\n",
1469 p->p_pid, curlwp->l_lid, l->l_lid));
1470 LIST_INSERT_HEAD(&vp->savp_lwpcache, l, l_sibling);
1471 vp->savp_ncached++;
1472 /* XXX unlock */
1473 }
1474
1475 /*
1476 * sa_getcachelwp
1477 * Fetch a LWP from the cache.
1478 * Scheduler lock held on entry and exit.
1479 */
1480 struct lwp *
1481 sa_getcachelwp(struct sadata_vp *vp)
1482 {
1483 struct lwp *l;
1484 struct proc *p;
1485
1486 SCHED_ASSERT_LOCKED();
1487
1488 l = NULL;
1489 /* XXX lock sadata */
1490 if (vp->savp_ncached > 0) {
1491 vp->savp_ncached--;
1492 l = LIST_FIRST(&vp->savp_lwpcache);
1493 LIST_REMOVE(l, l_sibling);
1494 p = l->l_proc;
1495 LIST_INSERT_HEAD(&p->p_lwps, l, l_sibling);
1496 p->p_nlwps++;
1497 DPRINTFN(5,("sa_getcachelwp(%d.%d) Got LWP %d from cache.\n",
1498 p->p_pid, curlwp->l_lid, l->l_lid));
1499 }
1500 /* XXX unlock */
1501 return l;
1502 }
1503
1504
1505 void
1506 sa_unblock_userret(struct lwp *l)
1507 {
1508 struct proc *p;
1509 struct lwp *l2;
1510 struct sadata *sa;
1511 struct sadata_vp *vp;
1512 struct sadata_upcall *sau;
1513 struct sastack *sast;
1514 int f, s;
1515
1516 p = l->l_proc;
1517 sa = p->p_sa;
1518 vp = l->l_savp;
1519
1520 if (p->p_flag & P_WEXIT)
1521 return;
1522
1523 SCHED_ASSERT_UNLOCKED();
1524
1525 KERNEL_PROC_LOCK(l);
1526 SA_LWP_STATE_LOCK(l, f);
1527
1528 DPRINTFN(7,("sa_unblock_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
1529 l->l_flag));
1530
1531 sa_setwoken(l);
1532 /* maybe NOTREACHED */
1533
1534 SCHED_LOCK(s);
1535 if (l != vp->savp_lwp) {
1536 /* Invoke an "unblocked" upcall */
1537 DPRINTFN(8,("sa_unblock_userret(%d.%d) unblocking\n",
1538 p->p_pid, l->l_lid));
1539
1540 l2 = sa_vp_repossess(l);
1541
1542 SCHED_UNLOCK(s);
1543
1544 if (l2 == NULL)
1545 lwp_exit(l);
1546
1547 sast = sa_getstack(sa);
1548 if (p->p_flag & P_WEXIT)
1549 lwp_exit(l);
1550
1551 sau = sadata_upcall_alloc(1);
1552 if (p->p_flag & P_WEXIT) {
1553 sadata_upcall_free(sau);
1554 lwp_exit(l);
1555 }
1556
1557 KDASSERT(l2 != NULL);
1558 PHOLD(l2);
1559
1560 KDASSERT(sast != NULL);
1561 DPRINTFN(9,("sa_unblock_userret(%d.%d) using stack %p\n",
1562 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
1563
1564 /*
1565 * Defer saving the event lwp's state because a
1566 * PREEMPT upcall could be on the queue already.
1567 */
1568 sa_upcall0(sau, SA_UPCALL_UNBLOCKED | SA_UPCALL_DEFER_EVENT,
1569 l, l2, 0, NULL, NULL);
1570 sau->sau_stack = sast->sast_stack;
1571
1572 SCHED_LOCK(s);
1573 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
1574 l->l_flag |= L_SA_UPCALL;
1575 l->l_flag &= ~L_SA_BLOCKING;
1576 sa_putcachelwp(p, l2);
1577 }
1578 SCHED_UNLOCK(s);
1579
1580 SA_LWP_STATE_UNLOCK(l, f);
1581 KERNEL_PROC_UNLOCK(l);
1582 }
1583
1584 /*
1585 * sa_upcall_userret
1586 * We are about to exit the kernel and return to userland, and
1587 * userret() noticed we have upcalls pending. So deliver them.
1588 *
1589 * This is the place where unblocking upcalls get generated. We
1590 * allocate the stack & upcall event here. We may block doing so, but
1591 * we lock our LWP state (clear L_SA for the moment) while doing so.
1592 *
1593 * In the case of delivering multiple upcall events, we will end up
1594 * writing multiple stacks out to userland at once. The last one we send
1595 * out will be the first one run, then it will notice the others and
1596 * run them.
1597 *
1598 * No locks held on entry or exit. We lock the scheduler during processing.
1599 */
1600 void
1601 sa_upcall_userret(struct lwp *l)
1602 {
1603 struct lwp *l2;
1604 struct proc *p;
1605 struct sadata *sa;
1606 struct sadata_vp *vp;
1607 struct sadata_upcall *sau;
1608 struct sastack *sast;
1609 int f, s;
1610
1611 p = l->l_proc;
1612 sa = p->p_sa;
1613 vp = l->l_savp;
1614
1615 SCHED_ASSERT_UNLOCKED();
1616
1617 KERNEL_PROC_LOCK(l);
1618 SA_LWP_STATE_LOCK(l, f);
1619
1620 DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid,
1621 l->l_flag));
1622
1623 KDASSERT((l->l_flag & L_SA_BLOCKING) == 0);
1624
1625 sast = NULL;
1626 if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL) {
1627 sast = sa_getstack(sa);
1628 if (sast == NULL) {
1629 SA_LWP_STATE_UNLOCK(l, f);
1630 KERNEL_PROC_UNLOCK(l);
1631 preempt(1);
1632 return;
1633 }
1634 }
1635 SCHED_LOCK(s);
1636 if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL &&
1637 sast != NULL) {
1638 /* Invoke an "unblocked" upcall */
1639 l2 = vp->savp_wokenq_head;
1640 vp->savp_wokenq_head = l2->l_forw;
1641
1642 DPRINTFN(9,("sa_upcall_userret(%d.%d) using stack %p\n",
1643 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp));
1644
1645 SCHED_UNLOCK(s);
1646
1647 if (p->p_flag & P_WEXIT)
1648 lwp_exit(l);
1649
1650 DPRINTFN(8,("sa_upcall_userret(%d.%d) unblocking %d\n",
1651 p->p_pid, l->l_lid, l2->l_lid));
1652
1653 sau = sadata_upcall_alloc(1);
1654 if (p->p_flag & P_WEXIT) {
1655 sadata_upcall_free(sau);
1656 lwp_exit(l);
1657 }
1658
1659 sa_upcall0(sau, SA_UPCALL_UNBLOCKED, l2, l, 0, NULL, NULL);
1660 sau->sau_stack = sast->sast_stack;
1661
1662 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next);
1663
1664 l2->l_flag &= ~L_SA_BLOCKING;
1665 SCHED_LOCK(s);
1666 sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */
1667 SCHED_UNLOCK(s);
1668 } else {
1669 SCHED_UNLOCK(s);
1670 if (sast)
1671 sa_setstackfree(sast, sa);
1672 }
1673
1674 KDASSERT(vp->savp_lwp == l);
1675
1676 while (!SIMPLEQ_EMPTY(&vp->savp_upcalls))
1677 sa_makeupcalls(l);
1678
1679 if (vp->savp_wokenq_head == NULL)
1680 l->l_flag &= ~L_SA_UPCALL;
1681
1682 SA_LWP_STATE_UNLOCK(l, f);
1683 KERNEL_PROC_UNLOCK(l);
1684 return;
1685 }
1686
1687 #define SACOPYOUT(sae, type, kp, up) \
1688 (((sae)->sae_sacopyout != NULL) ? \
1689 (*(sae)->sae_sacopyout)((type), (kp), (void *)(up)) : \
1690 copyout((kp), (void *)(up), sizeof(*(kp))))
1691
1692 /*
1693 * sa_makeupcalls
1694 * We're delivering the first upcall on lwp l, so
1695 * copy everything out. We assigned the stack for this upcall
1696 * when we enqueued it.
1697 *
1698 * KERNEL_PROC_LOCK should be held on entry and exit, and
1699 * SA_LWP_STATE should also be locked (L_SA temporarily disabled).
1700 *
1701 * If the enqueued event was DEFERRED, this is the time when we set
1702 * up the upcall event's state.
1703 */
1704 static inline void
1705 sa_makeupcalls(struct lwp *l)
1706 {
1707 struct lwp *l2, *eventq;
1708 struct proc *p;
1709 const struct sa_emul *sae;
1710 struct sadata *sa;
1711 struct sadata_vp *vp;
1712 uintptr_t sapp, sap;
1713 struct sa_t self_sa;
1714 struct sa_t *sas[3];
1715 #ifdef KTRACE
1716 struct sa_t **ksapp = NULL;
1717 #endif
1718 struct sadata_upcall *sau;
1719 void *stack, *ap;
1720 union sau_state *e_ss;
1721 ucontext_t *kup, *up;
1722 size_t sz, ucsize;
1723 int i, nint, nevents, s, type, error;
1724
1725 p = l->l_proc;
1726 sae = p->p_emul->e_sa;
1727 sa = p->p_sa;
1728 vp = l->l_savp;
1729 ucsize = sae->sae_ucsize;
1730
1731 sau = SIMPLEQ_FIRST(&vp->savp_upcalls);
1732 SIMPLEQ_REMOVE_HEAD(&vp->savp_upcalls, sau_next);
1733
1734 if (sau->sau_flags & SAU_FLAG_DEFERRED_EVENT)
1735 sa_upcall_getstate(&sau->sau_event,
1736 sau->sau_event.ss_deferred.ss_lwp);
1737 if (sau->sau_flags & SAU_FLAG_DEFERRED_INTERRUPTED)
1738 sa_upcall_getstate(&sau->sau_interrupted,
1739 sau->sau_interrupted.ss_deferred.ss_lwp);
1740
1741 #ifdef __MACHINE_STACK_GROWS_UP
1742 stack = sau->sau_stack.ss_sp;
1743 #else
1744 stack = (caddr_t)sau->sau_stack.ss_sp + sau->sau_stack.ss_size;
1745 #endif
1746 stack = STACK_ALIGN(stack, ALIGNBYTES);
1747
1748 self_sa.sa_id = l->l_lid;
1749 self_sa.sa_cpu = vp->savp_id;
1750 sas[0] = &self_sa;
1751 nevents = 0;
1752 nint = 0;
1753 if (sau->sau_event.ss_captured.ss_sa.sa_context != NULL) {
1754 if (copyout(&sau->sau_event.ss_captured.ss_ctx,
1755 sau->sau_event.ss_captured.ss_sa.sa_context,
1756 ucsize) != 0) {
1757 #ifdef DIAGNOSTIC
1758 printf("sa_makeupcalls(%d.%d): couldn't copyout"
1759 " context of event LWP %d\n",
1760 p->p_pid, l->l_lid,
1761 sau->sau_event.ss_captured.ss_sa.sa_id);
1762 #endif
1763 sigexit(l, SIGILL);
1764 /* NOTREACHED */
1765 }
1766 sas[1] = &sau->sau_event.ss_captured.ss_sa;
1767 nevents = 1;
1768 }
1769 if (sau->sau_interrupted.ss_captured.ss_sa.sa_context != NULL) {
1770 KDASSERT(sau->sau_interrupted.ss_captured.ss_sa.sa_context !=
1771 sau->sau_event.ss_captured.ss_sa.sa_context);
1772 if (copyout(&sau->sau_interrupted.ss_captured.ss_ctx,
1773 sau->sau_interrupted.ss_captured.ss_sa.sa_context,
1774 ucsize) != 0) {
1775 #ifdef DIAGNOSTIC
1776 printf("sa_makeupcalls(%d.%d): couldn't copyout"
1777 " context of interrupted LWP %d\n",
1778 p->p_pid, l->l_lid,
1779 sau->sau_interrupted.ss_captured.ss_sa.sa_id);
1780 #endif
1781 sigexit(l, SIGILL);
1782 /* NOTREACHED */
1783 }
1784 sas[2] = &sau->sau_interrupted.ss_captured.ss_sa;
1785 nint = 1;
1786 }
1787 eventq = NULL;
1788 if (sau->sau_type == SA_UPCALL_UNBLOCKED) {
1789 SCHED_LOCK(s);
1790 eventq = vp->savp_wokenq_head;
1791 vp->savp_wokenq_head = NULL;
1792 SCHED_UNLOCK(s);
1793 l2 = eventq;
1794 while (l2 != NULL) {
1795 nevents++;
1796 l2 = l2->l_forw;
1797 }
1798 }
1799
1800 /* Copy out the activation's ucontext */
1801 up = (void *)STACK_ALLOC(stack, ucsize);
1802 stack = STACK_GROW(stack, ucsize);
1803 kup = kmem_zalloc(sizeof(*kup), KM_SLEEP);
1804 kup->uc_stack = sau->sau_stack;
1805 kup->uc_flags = _UC_STACK;
1806 error = SACOPYOUT(sae, SAOUT_UCONTEXT, kup, up);
1807 kmem_free(kup, sizeof(*kup));
1808 if (error) {
1809 sadata_upcall_free(sau);
1810 #ifdef DIAGNOSTIC
1811 printf("sa_makeupcalls: couldn't copyout activation"
1812 " ucontext for %d.%d to %p\n", l->l_proc->p_pid, l->l_lid,
1813 up);
1814 #endif
1815 sigexit(l, SIGILL);
1816 /* NOTREACHED */
1817 }
1818 sas[0]->sa_context = up;
1819
1820 /* Next, copy out the sa_t's and pointers to them. */
1821
1822 sz = (1 + nevents + nint) * sae->sae_sasize;
1823 sap = (uintptr_t)STACK_ALLOC(stack, sz);
1824 sap += sz;
1825 stack = STACK_GROW(stack, sz);
1826
1827 sz = (1 + nevents + nint) * sae->sae_sapsize;
1828 sapp = (uintptr_t)STACK_ALLOC(stack, sz);
1829 sapp += sz;
1830 stack = STACK_GROW(stack, sz);
1831
1832 #ifdef KTRACE
1833 if (KTRPOINT(p, KTR_SAUPCALL))
1834 ksapp = kmem_alloc(sizeof(struct sa_t *) * (nevents + nint + 1),
1835 KM_SLEEP);
1836 #endif
1837 KDASSERT(nint <= 1);
1838 e_ss = NULL;
1839 for (i = nevents + nint; i >= 0; i--) {
1840 struct sa_t *sasp;
1841
1842 sap -= sae->sae_sasize;
1843 sapp -= sae->sae_sapsize;
1844 error = 0;
1845 if (i == 1 + nevents) /* interrupted sa */
1846 sasp = sas[2];
1847 else if (i <= 1) /* self_sa and event sa */
1848 sasp = sas[i];
1849 else { /* extra sas */
1850 KDASSERT(sau->sau_type == SA_UPCALL_UNBLOCKED);
1851 KDASSERT(eventq != NULL);
1852 l2 = eventq;
1853 KDASSERT(l2 != NULL);
1854 eventq = l2->l_forw;
1855 DPRINTFN(8,
1856 ("sa_makeupcalls(%d.%d) unblocking extra %d\n",
1857 p->p_pid, l->l_lid, l2->l_lid));
1858 if (e_ss == NULL) {
1859 e_ss = kmem_alloc(sizeof(*e_ss), KM_SLEEP);
1860 }
1861 sa_upcall_getstate(e_ss, l2);
1862 SCHED_LOCK(s);
1863 l2->l_flag &= ~L_SA_BLOCKING;
1864 sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */
1865 SCHED_UNLOCK(s);
1866
1867 error = copyout(&e_ss->ss_captured.ss_ctx,
1868 e_ss->ss_captured.ss_sa.sa_context, ucsize);
1869 sasp = &e_ss->ss_captured.ss_sa;
1870 }
1871 if (error != 0 ||
1872 SACOPYOUT(sae, SAOUT_SA_T, sasp, sap) ||
1873 SACOPYOUT(sae, SAOUT_SAP_T, &sap, sapp)) {
1874 /* Copying onto the stack didn't work. Die. */
1875 sadata_upcall_free(sau);
1876 #ifdef DIAGNOSTIC
1877 printf("sa_makeupcalls(%d.%d): couldn't copyout\n",
1878 p->p_pid, l->l_lid);
1879 #endif
1880 if (e_ss != NULL) {
1881 kmem_free(e_ss, sizeof(*e_ss));
1882 }
1883 goto fail;
1884 }
1885 #ifdef KTRACE
1886 if (KTRPOINT(p, KTR_SAUPCALL))
1887 ksapp[i] = sasp;
1888 #endif
1889
1890 }
1891 if (e_ss != NULL) {
1892 kmem_free(e_ss, sizeof(*e_ss));
1893 }
1894 KDASSERT(eventq == NULL);
1895
1896 /* Copy out the arg, if any */
1897 /* xxx assume alignment works out; everything so far has been
1898 * a structure, so...
1899 */
1900 if (sau->sau_arg) {
1901 ap = STACK_ALLOC(stack, sau->sau_argsize);
1902 stack = STACK_GROW(stack, sau->sau_argsize);
1903 if (copyout(sau->sau_arg, ap, sau->sau_argsize) != 0) {
1904 /* Copying onto the stack didn't work. Die. */
1905 sadata_upcall_free(sau);
1906 #ifdef DIAGNOSTIC
1907 printf("sa_makeupcalls(%d.%d): couldn't copyout"
1908 " sadata_upcall arg %p size %ld to %p \n",
1909 p->p_pid, l->l_lid,
1910 sau->sau_arg, (long) sau->sau_argsize, ap);
1911 #endif
1912 goto fail;
1913 }
1914 } else {
1915 ap = NULL;
1916 #ifdef __hppa__
1917 stack = STACK_ALIGN(stack, HPPA_FRAME_SIZE);
1918 #endif
1919 }
1920 type = sau->sau_type;
1921
1922 if (vp->savp_sleeper_upcall == NULL)
1923 vp->savp_sleeper_upcall = sau;
1924 else
1925 sadata_upcall_free(sau);
1926
1927 DPRINTFN(7,("sa_makeupcalls(%d.%d): type %d\n", p->p_pid,
1928 l->l_lid, type));
1929
1930 #ifdef KTRACE
1931 if (KTRPOINT(p, KTR_SAUPCALL)) {
1932 ktrsaupcall(l, type, nevents, nint, (void *)sapp, ap, ksapp);
1933 kmem_free(ksapp, sizeof(struct sa_t *) * (nevents + nint + 1));
1934 }
1935 #endif
1936 (*sae->sae_upcall)(l, type, nevents, nint, (void *)sapp, ap, stack,
1937 sa->sa_upcall);
1938
1939 l->l_flag &= ~L_SA_YIELD;
1940 return;
1941
1942 fail:
1943 #ifdef KTRACE
1944 if (KTRPOINT(p, KTR_SAUPCALL))
1945 kmem_free(ksapp, sizeof(struct sa_t) * (nevents + nint + 1));
1946 #endif
1947 sigexit(l, SIGILL);
1948 /* NOTREACHED */
1949 }
1950
1951 static void
1952 sa_setwoken(struct lwp *l)
1953 {
1954 struct lwp *l2, *vp_lwp;
1955 struct proc *p;
1956 struct sadata *sa;
1957 struct sadata_vp *vp;
1958 int s;
1959
1960 SCHED_LOCK(s);
1961
1962 if ((l->l_flag & L_SA_BLOCKING) == 0) {
1963 SCHED_UNLOCK(s);
1964 return;
1965 }
1966
1967 p = l->l_proc;
1968 sa = p->p_sa;
1969 vp = l->l_savp;
1970 vp_lwp = vp->savp_lwp;
1971 l2 = NULL;
1972
1973 KDASSERT(vp_lwp != NULL);
1974 DPRINTFN(3,("sa_setwoken(%d.%d) woken, flags %x, vp %d\n",
1975 l->l_proc->p_pid, l->l_lid, l->l_flag,
1976 vp_lwp->l_lid));
1977
1978 #if notyet
1979 if (vp_lwp->l_flag & L_SA_IDLE) {
1980 KDASSERT((vp_lwp->l_flag & L_SA_UPCALL) == 0);
1981 KDASSERT(vp->savp_wokenq_head == NULL);
1982 DPRINTFN(3,
1983 ("sa_setwoken(%d.%d) repossess: idle vp_lwp %d state %d\n",
1984 l->l_proc->p_pid, l->l_lid,
1985 vp_lwp->l_lid, vp_lwp->l_stat));
1986 vp_lwp->l_flag &= ~L_SA_IDLE;
1987 SCHED_UNLOCK(s);
1988 return;
1989 }
1990 #endif
1991
1992 DPRINTFN(3,("sa_setwoken(%d.%d) put on wokenq: vp_lwp %d state %d\n",
1993 l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid,
1994 vp_lwp->l_stat));
1995
1996 PHOLD(l);
1997 if (vp->savp_wokenq_head == NULL)
1998 vp->savp_wokenq_head = l;
1999 else
2000 *vp->savp_wokenq_tailp = l;
2001 *(vp->savp_wokenq_tailp = &l->l_forw) = NULL;
2002
2003 switch (vp_lwp->l_stat) {
2004 case LSONPROC:
2005 if (vp_lwp->l_flag & L_SA_UPCALL)
2006 break;
2007 vp_lwp->l_flag |= L_SA_UPCALL;
2008 if (vp_lwp->l_flag & L_SA_YIELD)
2009 break;
2010 /* XXX IPI vp_lwp->l_cpu */
2011 break;
2012 case LSSLEEP:
2013 if (vp_lwp->l_flag & L_SA_IDLE) {
2014 vp_lwp->l_flag &= ~L_SA_IDLE;
2015 vp_lwp->l_flag |= L_SA_UPCALL;
2016 setrunnable(vp_lwp);
2017 break;
2018 }
2019 vp_lwp->l_flag |= L_SA_UPCALL;
2020 break;
2021 case LSSUSPENDED:
2022 #ifdef DIAGNOSTIC
2023 printf("sa_setwoken(%d.%d) vp lwp %d LSSUSPENDED\n",
2024 l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid);
2025 #endif
2026 break;
2027 case LSSTOP:
2028 vp_lwp->l_flag |= L_SA_UPCALL;
2029 break;
2030 case LSRUN:
2031 if (vp_lwp->l_flag & L_SA_UPCALL)
2032 break;
2033 vp_lwp->l_flag |= L_SA_UPCALL;
2034 if (vp_lwp->l_flag & L_SA_YIELD)
2035 break;
2036 if (vp_lwp->l_slptime > 1) {
2037 void updatepri(struct lwp *);
2038 updatepri(vp_lwp);
2039 }
2040 vp_lwp->l_slptime = 0;
2041 if (vp_lwp->l_flag & L_INMEM) {
2042 if (vp_lwp->l_cpu == curcpu())
2043 l2 = vp_lwp;
2044 else
2045 need_resched(vp_lwp->l_cpu);
2046 } else
2047 sched_wakeup(&proc0);
2048 break;
2049 default:
2050 panic("sa_vp LWP not sleeping/onproc/runnable");
2051 }
2052
2053 l->l_stat = LSSUSPENDED;
2054 p->p_nrlwps--;
2055 mi_switch(l, l2);
2056 /* maybe NOTREACHED */
2057 SCHED_ASSERT_UNLOCKED();
2058 splx(s);
2059 if (p->p_flag & P_WEXIT)
2060 lwp_exit(l);
2061 }
2062
2063 static struct lwp *
2064 sa_vp_repossess(struct lwp *l)
2065 {
2066 struct lwp *l2;
2067 struct proc *p = l->l_proc;
2068 struct sadata_vp *vp = l->l_savp;
2069
2070 SCHED_ASSERT_LOCKED();
2071
2072 /*
2073 * Put ourselves on the virtual processor and note that the
2074 * previous occupant of that position was interrupted.
2075 */
2076 l2 = vp->savp_lwp;
2077 vp->savp_lwp = l;
2078 if (l2) {
2079 if (l2->l_flag & L_SA_YIELD)
2080 l2->l_flag &= ~(L_SA_YIELD|L_SA_IDLE);
2081
2082 DPRINTFN(1,("sa_vp_repossess(%d.%d) vp lwp %d state %d\n",
2083 p->p_pid, l->l_lid, l2->l_lid, l2->l_stat));
2084
2085 KDASSERT(l2 != l);
2086 switch (l2->l_stat) {
2087 case LSRUN:
2088 remrunqueue(l2);
2089 p->p_nrlwps--;
2090 break;
2091 case LSSLEEP:
2092 unsleep(l2);
2093 l2->l_flag &= ~L_SINTR;
2094 break;
2095 case LSSUSPENDED:
2096 #ifdef DIAGNOSTIC
2097 printf("sa_vp_repossess(%d.%d) vp lwp %d LSSUSPENDED\n",
2098 l->l_proc->p_pid, l->l_lid, l2->l_lid);
2099 #endif
2100 break;
2101 #ifdef DIAGNOSTIC
2102 default:
2103 panic("SA VP %d.%d is in state %d, not running"
2104 " or sleeping\n", p->p_pid, l2->l_lid,
2105 l2->l_stat);
2106 #endif
2107 }
2108 l2->l_stat = LSSUSPENDED;
2109 }
2110 return l2;
2111 }
2112
2113
2114
2115 #ifdef DEBUG
2116 int debug_print_sa(struct proc *);
2117 int debug_print_lwp(struct lwp *);
2118 int debug_print_proc(int);
2119
2120 int
2121 debug_print_proc(int pid)
2122 {
2123 struct proc *p;
2124
2125 p = pfind(pid);
2126 if (p == NULL)
2127 printf("No process %d\n", pid);
2128 else
2129 debug_print_sa(p);
2130
2131 return 0;
2132 }
2133
2134 int
2135 debug_print_sa(struct proc *p)
2136 {
2137 struct lwp *l;
2138 struct sadata *sa;
2139 struct sadata_vp *vp;
2140
2141 printf("Process %d (%s), state %d, address %p, flags %x\n",
2142 p->p_pid, p->p_comm, p->p_stat, p, p->p_flag);
2143 printf("LWPs: %d (%d running, %d zombies)\n",
2144 p->p_nlwps, p->p_nrlwps, p->p_nzlwps);
2145 LIST_FOREACH(l, &p->p_lwps, l_sibling)
2146 debug_print_lwp(l);
2147 sa = p->p_sa;
2148 if (sa) {
2149 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) {
2150 if (vp->savp_lwp)
2151 printf("SA VP: %d %s\n", vp->savp_lwp->l_lid,
2152 vp->savp_lwp->l_flag & L_SA_YIELD ?
2153 (vp->savp_lwp->l_flag & L_SA_IDLE ?
2154 "idle" : "yielding") : "");
2155 printf("SAs: %d cached LWPs\n", vp->savp_ncached);
2156 LIST_FOREACH(l, &vp->savp_lwpcache, l_sibling)
2157 debug_print_lwp(l);
2158 }
2159 }
2160
2161 return 0;
2162 }
2163
2164 int
2165 debug_print_lwp(struct lwp *l)
2166 {
2167
2168 printf("LWP %d address %p ", l->l_lid, l);
2169 printf("state %d flags %x ", l->l_stat, l->l_flag);
2170 if (l->l_wchan)
2171 printf("wait %p %s", l->l_wchan, l->l_wmesg);
2172 printf("\n");
2173
2174 return 0;
2175 }
2176
2177 #endif
Cache object: 0a3ed08972cdfb6baca2688ae27c9594
|