Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_sa.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: kern_sa.c,v 1.50.2.2 2005/03/20 11:58:47 tron Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.50.2.2 2005/03/20 11:58:47 tron Exp $"); 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/pool.h> 45 #include <sys/proc.h> 46 #include <sys/types.h> 47 #include <sys/ucontext.h> 48 #include <sys/malloc.h> 49 #include <sys/mount.h> 50 #include <sys/sa.h> 51 #include <sys/savar.h> 52 #include <sys/syscallargs.h> 53 54 #include <uvm/uvm_extern.h> 55 56 static struct sadata_vp *sa_newsavp(struct sadata *); 57 static __inline int sa_stackused(struct sastack *, struct sadata *); 58 static __inline void sa_setstackfree(struct sastack *, struct sadata *); 59 static struct sastack *sa_getstack(struct sadata *); 60 static __inline struct sastack *sa_getstack0(struct sadata *); 61 static __inline int sast_compare(struct sastack *, struct sastack *); 62 #ifdef MULTIPROCESSOR 63 static int sa_increaseconcurrency(struct lwp *, int); 64 #endif 65 static void sa_setwoken(struct lwp *); 66 static void sa_switchcall(void *); 67 static int sa_newcachelwp(struct lwp *); 68 static __inline void sa_makeupcalls(struct lwp *); 69 static struct lwp *sa_vp_repossess(struct lwp *l); 70 71 static __inline int sa_pagefault(struct lwp *, ucontext_t *); 72 73 static int sa_upcall0(struct lwp *, int, struct lwp *, struct lwp *, 74 size_t, void *, struct sadata_upcall *); 75 static void sa_upcall_getstate(union sau_state *, struct lwp *); 76 77 MALLOC_DEFINE(M_SA, "sa", "Scheduler activations"); 78 79 #define SA_DEBUG 80 81 #ifdef SA_DEBUG 82 #define DPRINTF(x) do { if (sadebug) printf x; } while (0) 83 #define DPRINTFN(n,x) do { if (sadebug & (1<<(n-1))) printf x; } while (0) 84 int sadebug = 0; 85 #else 86 #define DPRINTF(x) 87 #define DPRINTFN(n,x) 88 #endif 89 90 91 #define SA_LWP_STATE_LOCK(l, f) do { \ 92 (f) = (l)->l_flag; \ 93 (l)->l_flag &= ~L_SA; \ 94 } while (/*CONSTCOND*/ 0) 95 96 #define SA_LWP_STATE_UNLOCK(l, f) do { \ 97 (l)->l_flag |= (f) & L_SA; \ 98 } while (/*CONSTCOND*/ 0) 99 100 SPLAY_PROTOTYPE(sasttree, sastack, sast_node, sast_compare); 101 SPLAY_GENERATE(sasttree, sastack, sast_node, sast_compare); 102 103 104 /* 105 * sadata_upcall_alloc: 106 * 107 * Allocate an sadata_upcall structure. 108 */ 109 struct sadata_upcall * 110 sadata_upcall_alloc(int waitok) 111 { 112 113 /* XXX zero the memory? */ 114 return (pool_get(&saupcall_pool, waitok ? PR_WAITOK : PR_NOWAIT)); 115 } 116 117 /* 118 * sadata_upcall_free: 119 * 120 * Free an sadata_upcall structure, and any associated 121 * argument data. 122 */ 123 void 124 sadata_upcall_free(struct sadata_upcall *sau) 125 { 126 extern struct pool siginfo_pool; /* XXX Ew. */ 127 128 /* 129 * XXX We have to know what the origin of sau_arg is 130 * XXX in order to do the right thing, here. Sucks 131 * XXX to be a non-garbage-collecting kernel. 132 */ 133 if (sau->sau_arg) { 134 switch (sau->sau_type) { 135 case SA_UPCALL_SIGNAL: 136 case SA_UPCALL_SIGEV: 137 pool_put(&siginfo_pool, sau->sau_arg); 138 break; 139 default: 140 panic("sadata_free: unknown type of sau_arg: %d", 141 sau->sau_type); 142 } 143 } 144 145 pool_put(&saupcall_pool, sau); 146 } 147 148 static struct sadata_vp * 149 sa_newsavp(struct sadata *sa) 150 { 151 struct sadata_vp *vp, *qvp; 152 153 /* Allocate virtual processor data structure */ 154 vp = pool_get(&savp_pool, PR_WAITOK); 155 /* Initialize. */ 156 memset(vp, 0, sizeof(*vp)); 157 simple_lock_init(&vp->savp_lock); 158 vp->savp_lwp = NULL; 159 vp->savp_wokenq_head = NULL; 160 vp->savp_faultaddr = 0; 161 vp->savp_ofaultaddr = 0; 162 LIST_INIT(&vp->savp_lwpcache); 163 vp->savp_ncached = 0; 164 SIMPLEQ_INIT(&vp->savp_upcalls); 165 166 simple_lock(&sa->sa_lock); 167 /* find first free savp_id and add vp to sorted slist */ 168 if (SLIST_EMPTY(&sa->sa_vps) || 169 SLIST_FIRST(&sa->sa_vps)->savp_id != 0) { 170 vp->savp_id = 0; 171 SLIST_INSERT_HEAD(&sa->sa_vps, vp, savp_next); 172 } else { 173 SLIST_FOREACH(qvp, &sa->sa_vps, savp_next) { 174 if (SLIST_NEXT(qvp, savp_next) == NULL || 175 SLIST_NEXT(qvp, savp_next)->savp_id != 176 qvp->savp_id + 1) 177 break; 178 } 179 vp->savp_id = qvp->savp_id + 1; 180 SLIST_INSERT_AFTER(qvp, vp, savp_next); 181 } 182 simple_unlock(&sa->sa_lock); 183 184 return (vp); 185 } 186 187 int 188 sys_sa_register(struct lwp *l, void *v, register_t *retval) 189 { 190 struct sys_sa_register_args /* { 191 syscallarg(sa_upcall_t) new; 192 syscallarg(sa_upcall_t *) old; 193 syscallarg(int) flags; 194 syscallarg(ssize_t) stackinfo_offset; 195 } */ *uap = v; 196 struct proc *p = l->l_proc; 197 struct sadata *sa; 198 sa_upcall_t prev; 199 int error; 200 201 if (p->p_sa == NULL) { 202 /* Allocate scheduler activations data structure */ 203 sa = pool_get(&sadata_pool, PR_WAITOK); 204 /* Initialize. */ 205 memset(sa, 0, sizeof(*sa)); 206 simple_lock_init(&sa->sa_lock); 207 sa->sa_flag = SCARG(uap, flags) & SA_FLAG_ALL; 208 sa->sa_maxconcurrency = 1; 209 sa->sa_concurrency = 1; 210 SPLAY_INIT(&sa->sa_stackstree); 211 sa->sa_stacknext = NULL; 212 if (SCARG(uap, flags) & SA_FLAG_STACKINFO) 213 sa->sa_stackinfo_offset = SCARG(uap, stackinfo_offset); 214 else 215 sa->sa_stackinfo_offset = 0; 216 sa->sa_nstacks = 0; 217 SLIST_INIT(&sa->sa_vps); 218 p->p_sa = sa; 219 KASSERT(l->l_savp == NULL); 220 } 221 if (l->l_savp == NULL) { 222 l->l_savp = sa_newsavp(p->p_sa); 223 sa_newcachelwp(l); 224 } 225 226 prev = p->p_sa->sa_upcall; 227 p->p_sa->sa_upcall = SCARG(uap, new); 228 if (SCARG(uap, old)) { 229 error = copyout(&prev, SCARG(uap, old), 230 sizeof(prev)); 231 if (error) 232 return (error); 233 } 234 235 return (0); 236 } 237 238 void 239 sa_release(struct proc *p) 240 { 241 struct sadata *sa; 242 struct sastack *sast, *next; 243 struct sadata_vp *vp; 244 struct lwp *l; 245 246 sa = p->p_sa; 247 KDASSERT(sa != NULL); 248 KASSERT(p->p_nlwps <= 1); 249 250 for (sast = SPLAY_MIN(sasttree, &sa->sa_stackstree); sast != NULL; 251 sast = next) { 252 next = SPLAY_NEXT(sasttree, &sa->sa_stackstree, sast); 253 SPLAY_REMOVE(sasttree, &sa->sa_stackstree, sast); 254 pool_put(&sastack_pool, sast); 255 } 256 257 p->p_flag &= ~P_SA; 258 while ((vp = SLIST_FIRST(&p->p_sa->sa_vps)) != NULL) { 259 SLIST_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next); 260 pool_put(&savp_pool, vp); 261 } 262 pool_put(&sadata_pool, sa); 263 p->p_sa = NULL; 264 l = LIST_FIRST(&p->p_lwps); 265 if (l) { 266 KASSERT(LIST_NEXT(l, l_sibling) == NULL); 267 l->l_savp = NULL; 268 } 269 } 270 271 272 static __inline int 273 sa_stackused(struct sastack *sast, struct sadata *sa) 274 { 275 unsigned int gen; 276 277 if (copyin((void *)&((struct sa_stackinfo_t *) 278 ((char *)sast->sast_stack.ss_sp + 279 sa->sa_stackinfo_offset))->sasi_stackgen, 280 &gen, sizeof(unsigned int)) != 0) { 281 #ifdef DIAGNOSTIC 282 printf("sa_stackused: couldn't copyin sasi_stackgen"); 283 #endif 284 sigexit(curlwp, SIGILL); 285 /* NOTREACHED */ 286 } 287 return (sast->sast_gen != gen); 288 } 289 290 static __inline void 291 sa_setstackfree(struct sastack *sast, struct sadata *sa) 292 { 293 294 if (copyin((void *)&((struct sa_stackinfo_t *) 295 ((char *)sast->sast_stack.ss_sp + 296 sa->sa_stackinfo_offset))->sasi_stackgen, 297 &sast->sast_gen, sizeof(unsigned int)) != 0) { 298 #ifdef DIAGNOSTIC 299 printf("sa_setstackfree: couldn't copyin sasi_stackgen"); 300 #endif 301 sigexit(curlwp, SIGILL); 302 /* NOTREACHED */ 303 } 304 } 305 306 /* 307 * Find next free stack, starting at sa->sa_stacknext. 308 */ 309 static struct sastack * 310 sa_getstack(struct sadata *sa) 311 { 312 struct sastack *sast; 313 314 SCHED_ASSERT_UNLOCKED(); 315 316 if ((sast = sa->sa_stacknext) == NULL || sa_stackused(sast, sa)) 317 sast = sa_getstack0(sa); 318 319 if (sast == NULL) 320 return NULL; 321 322 sast->sast_gen++; 323 324 return sast; 325 } 326 327 static __inline struct sastack * 328 sa_getstack0(struct sadata *sa) 329 { 330 struct sastack *start; 331 332 if (sa->sa_stacknext == NULL) { 333 sa->sa_stacknext = SPLAY_MIN(sasttree, &sa->sa_stackstree); 334 if (sa->sa_stacknext == NULL) 335 return NULL; 336 } 337 start = sa->sa_stacknext; 338 339 while (sa_stackused(sa->sa_stacknext, sa)) { 340 sa->sa_stacknext = SPLAY_NEXT(sasttree, &sa->sa_stackstree, 341 sa->sa_stacknext); 342 if (sa->sa_stacknext == NULL) 343 sa->sa_stacknext = SPLAY_MIN(sasttree, 344 &sa->sa_stackstree); 345 if (sa->sa_stacknext == start) 346 return NULL; 347 } 348 return sa->sa_stacknext; 349 } 350 351 static __inline int 352 sast_compare(struct sastack *a, struct sastack *b) 353 { 354 if ((vaddr_t)a->sast_stack.ss_sp + a->sast_stack.ss_size <= 355 (vaddr_t)b->sast_stack.ss_sp) 356 return (-1); 357 if ((vaddr_t)a->sast_stack.ss_sp >= 358 (vaddr_t)b->sast_stack.ss_sp + b->sast_stack.ss_size) 359 return (1); 360 return (0); 361 } 362 363 int 364 sys_sa_stacks(struct lwp *l, void *v, register_t *retval) 365 { 366 struct sys_sa_stacks_args /* { 367 syscallarg(int) num; 368 syscallarg(stack_t *) stacks; 369 } */ *uap = v; 370 struct sadata *sa = l->l_proc->p_sa; 371 struct sastack *sast, newsast; 372 int count, error, f, i; 373 374 /* We have to be using scheduler activations */ 375 if (sa == NULL) 376 return (EINVAL); 377 378 count = SCARG(uap, num); 379 if (count < 0) 380 return (EINVAL); 381 382 SA_LWP_STATE_LOCK(l, f); 383 384 error = 0; 385 386 for (i = 0; i < count; i++) { 387 error = copyin(SCARG(uap, stacks) + i, &newsast.sast_stack, 388 sizeof(stack_t)); 389 if (error) { 390 count = i; 391 break; 392 } 393 if ((sast = SPLAY_FIND(sasttree, &sa->sa_stackstree, &newsast))) { 394 DPRINTFN(9, ("sa_stacks(%d.%d) returning stack %p\n", 395 l->l_proc->p_pid, l->l_lid, 396 newsast.sast_stack.ss_sp)); 397 if (sa_stackused(sast, sa) == 0) { 398 count = i; 399 error = EEXIST; 400 break; 401 } 402 } else if (sa->sa_nstacks >= SA_MAXNUMSTACKS * sa->sa_concurrency) { 403 DPRINTFN(9, ("sa_stacks(%d.%d) already using %d stacks\n", 404 l->l_proc->p_pid, l->l_lid, 405 SA_MAXNUMSTACKS * sa->sa_concurrency)); 406 count = i; 407 error = ENOMEM; 408 break; 409 } else { 410 DPRINTFN(9, ("sa_stacks(%d.%d) adding stack %p\n", 411 l->l_proc->p_pid, l->l_lid, 412 newsast.sast_stack.ss_sp)); 413 sast = pool_get(&sastack_pool, PR_WAITOK); 414 sast->sast_stack = newsast.sast_stack; 415 SPLAY_INSERT(sasttree, &sa->sa_stackstree, sast); 416 sa->sa_nstacks++; 417 } 418 sa_setstackfree(sast, sa); 419 } 420 421 SA_LWP_STATE_UNLOCK(l, f); 422 423 *retval = count; 424 return (error); 425 } 426 427 428 int 429 sys_sa_enable(struct lwp *l, void *v, register_t *retval) 430 { 431 struct proc *p = l->l_proc; 432 struct sadata *sa = p->p_sa; 433 struct sadata_vp *vp = l->l_savp; 434 int error; 435 436 DPRINTF(("sys_sa_enable(%d.%d)\n", l->l_proc->p_pid, 437 l->l_lid)); 438 439 /* We have to be using scheduler activations */ 440 if (sa == NULL || vp == NULL) 441 return (EINVAL); 442 443 if (p->p_flag & P_SA) /* Already running! */ 444 return (EBUSY); 445 446 error = sa_upcall(l, SA_UPCALL_NEWPROC, l, NULL, 0, NULL); 447 if (error) 448 return (error); 449 450 /* Assign this LWP to the virtual processor */ 451 vp->savp_lwp = l; 452 453 p->p_flag |= P_SA; 454 l->l_flag |= L_SA; /* We are now an activation LWP */ 455 456 /* This will not return to the place in user space it came from. */ 457 return (0); 458 } 459 460 461 #ifdef MULTIPROCESSOR 462 static int 463 sa_increaseconcurrency(struct lwp *l, int concurrency) 464 { 465 struct proc *p; 466 struct lwp *l2; 467 struct sadata *sa; 468 vaddr_t uaddr; 469 boolean_t inmem; 470 int addedconcurrency, error, s; 471 472 p = l->l_proc; 473 sa = p->p_sa; 474 475 addedconcurrency = 0; 476 simple_lock(&sa->sa_lock); 477 while (sa->sa_maxconcurrency < concurrency) { 478 sa->sa_maxconcurrency++; 479 sa->sa_concurrency++; 480 simple_unlock(&sa->sa_lock); 481 482 inmem = uvm_uarea_alloc(&uaddr); 483 if (__predict_false(uaddr == 0)) { 484 /* reset concurrency */ 485 simple_lock(&sa->sa_lock); 486 sa->sa_maxconcurrency--; 487 sa->sa_concurrency--; 488 simple_unlock(&sa->sa_lock); 489 return (addedconcurrency); 490 } else { 491 newlwp(l, p, uaddr, inmem, 0, NULL, 0, 492 child_return, 0, &l2); 493 l2->l_flag |= L_SA; 494 l2->l_savp = sa_newsavp(sa); 495 if (l2->l_savp) { 496 l2->l_savp->savp_lwp = l2; 497 cpu_setfunc(l2, sa_switchcall, NULL); 498 error = sa_upcall(l2, SA_UPCALL_NEWPROC, 499 NULL, NULL, 0, NULL); 500 if (error) { 501 /* free new savp */ 502 SLIST_REMOVE(&sa->sa_vps, l2->l_savp, 503 sadata_vp, savp_next); 504 pool_put(&savp_pool, l2->l_savp); 505 } 506 } else 507 error = 1; 508 if (error) { 509 /* put l2 into l's LWP cache */ 510 l2->l_savp = l->l_savp; 511 PHOLD(l2); 512 SCHED_LOCK(s); 513 sa_putcachelwp(p, l2); 514 SCHED_UNLOCK(s); 515 /* reset concurrency */ 516 simple_lock(&sa->sa_lock); 517 sa->sa_maxconcurrency--; 518 sa->sa_concurrency--; 519 simple_unlock(&sa->sa_lock); 520 return (addedconcurrency); 521 } 522 SCHED_LOCK(s); 523 setrunnable(l2); 524 SCHED_UNLOCK(s); 525 addedconcurrency++; 526 } 527 simple_lock(&sa->sa_lock); 528 } 529 simple_unlock(&sa->sa_lock); 530 531 return (addedconcurrency); 532 } 533 #endif 534 535 int 536 sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval) 537 { 538 struct sys_sa_setconcurrency_args /* { 539 syscallarg(int) concurrency; 540 } */ *uap = v; 541 struct sadata *sa = l->l_proc->p_sa; 542 #ifdef MULTIPROCESSOR 543 struct sadata_vp *vp = l->l_savp; 544 int ncpus, s; 545 struct cpu_info *ci; 546 CPU_INFO_ITERATOR cii; 547 #endif 548 549 DPRINTFN(11,("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid, 550 l->l_lid)); 551 552 /* We have to be using scheduler activations */ 553 if (sa == NULL) 554 return (EINVAL); 555 556 if (SCARG(uap, concurrency) < 1) 557 return (EINVAL); 558 559 *retval = 0; 560 /* 561 * Concurrency greater than the number of physical CPUs does 562 * not make sense. 563 * XXX Should we ever support hot-plug CPUs, this will need 564 * adjustment. 565 */ 566 #ifdef MULTIPROCESSOR 567 if (SCARG(uap, concurrency) > sa->sa_maxconcurrency) { 568 ncpus = 0; 569 for (CPU_INFO_FOREACH(cii, ci)) 570 ncpus++; 571 *retval += sa_increaseconcurrency(l, 572 min(SCARG(uap, concurrency), ncpus)); 573 } 574 #endif 575 576 DPRINTFN(11,("sys_sa_concurrency(%d.%d) want %d, have %d, max %d\n", 577 l->l_proc->p_pid, l->l_lid, SCARG(uap, concurrency), 578 sa->sa_concurrency, sa->sa_maxconcurrency)); 579 #ifdef MULTIPROCESSOR 580 if (SCARG(uap, concurrency) > sa->sa_concurrency) { 581 SCHED_LOCK(s); 582 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) { 583 if (vp->savp_lwp->l_flag & L_SA_IDLE) { 584 vp->savp_lwp->l_flag &= 585 ~(L_SA_IDLE|L_SA_YIELD|L_SINTR); 586 SCHED_UNLOCK(s); 587 DPRINTFN(11,("sys_sa_concurrency(%d.%d) NEWPROC vp %d\n", 588 l->l_proc->p_pid, l->l_lid, vp->savp_id)); 589 cpu_setfunc(vp->savp_lwp, sa_switchcall, NULL); 590 /* error = */ sa_upcall(vp->savp_lwp, SA_UPCALL_NEWPROC, 591 NULL, NULL, 0, NULL); 592 SCHED_LOCK(s); 593 sa->sa_concurrency++; 594 setrunnable(vp->savp_lwp); 595 KDASSERT((vp->savp_lwp->l_flag & L_SINTR) == 0); 596 (*retval)++; 597 } 598 if (sa->sa_concurrency == SCARG(uap, concurrency)) 599 break; 600 } 601 SCHED_UNLOCK(s); 602 } 603 #endif 604 605 return (0); 606 } 607 608 int 609 sys_sa_yield(struct lwp *l, void *v, register_t *retval) 610 { 611 struct proc *p = l->l_proc; 612 613 if (p->p_sa == NULL || !(p->p_flag & P_SA)) { 614 DPRINTFN(1,("sys_sa_yield(%d.%d) proc %p not SA (p_sa %p, flag %s)\n", 615 p->p_pid, l->l_lid, p, p->p_sa, p->p_flag & P_SA ? "T" : "F")); 616 return (EINVAL); 617 } 618 619 sa_yield(l); 620 621 return (EJUSTRETURN); 622 } 623 624 void 625 sa_yield(struct lwp *l) 626 { 627 struct proc *p = l->l_proc; 628 struct sadata *sa = p->p_sa; 629 struct sadata_vp *vp = l->l_savp; 630 int ret; 631 632 #if defined(MULTIPROCESSOR) 633 KDASSERT(l->l_flag & L_BIGLOCK); 634 #endif 635 636 if (vp->savp_lwp != l) { 637 /* 638 * We lost the VP on our way here, this happens for 639 * instance when we sleep in systrace. This will end 640 * in an SA_UNBLOCKED_UPCALL in sa_setwoken(). 641 */ 642 DPRINTFN(1,("sa_yield(%d.%d) lost VP\n", 643 p->p_pid, l->l_lid)); 644 KDASSERT(l->l_flag & L_SA_BLOCKING); 645 return; 646 } 647 648 /* 649 * If we're the last running LWP, stick around to recieve 650 * signals. 651 */ 652 KDASSERT((l->l_flag & L_SA_YIELD) == 0); 653 DPRINTFN(1,("sa_yield(%d.%d) going dormant\n", 654 p->p_pid, l->l_lid)); 655 /* 656 * A signal will probably wake us up. Worst case, the upcall 657 * happens and just causes the process to yield again. 658 */ 659 /* s = splsched(); */ /* Protect from timer expirations */ 660 KDASSERT(vp->savp_lwp == l); 661 /* 662 * If we were told to make an upcall or exit before 663 * the splsched(), make sure we process it instead of 664 * going to sleep. It might make more sense for this to 665 * be handled inside of tsleep.... 666 */ 667 ret = 0; 668 l->l_flag |= L_SA_YIELD; 669 if (l->l_flag & L_SA_UPCALL) { 670 /* KERNEL_PROC_UNLOCK(l); in upcallret() */ 671 upcallret(l); 672 KERNEL_PROC_LOCK(l); 673 } 674 while (l->l_flag & L_SA_YIELD) { 675 DPRINTFN(1,("sa_yield(%d.%d) really going dormant\n", 676 p->p_pid, l->l_lid)); 677 678 simple_lock(&sa->sa_lock); 679 sa->sa_concurrency--; 680 simple_unlock(&sa->sa_lock); 681 682 ret = tsleep((caddr_t) l, PUSER | PCATCH, "sawait", 0); 683 684 simple_lock(&sa->sa_lock); 685 sa->sa_concurrency++; 686 simple_unlock(&sa->sa_lock); 687 688 KDASSERT(vp->savp_lwp == l || p->p_flag & P_WEXIT); 689 690 /* KERNEL_PROC_UNLOCK(l); in upcallret() */ 691 upcallret(l); 692 KERNEL_PROC_LOCK(l); 693 } 694 /* splx(s); */ 695 DPRINTFN(1,("sa_yield(%d.%d) returned, ret %d, userret %p\n", 696 p->p_pid, l->l_lid, ret, p->p_userret)); 697 } 698 699 700 int 701 sys_sa_preempt(struct lwp *l, void *v, register_t *retval) 702 { 703 704 /* XXX Implement me. */ 705 return (ENOSYS); 706 } 707 708 709 /* XXX Hm, naming collision. */ 710 void 711 sa_preempt(struct lwp *l) 712 { 713 struct proc *p = l->l_proc; 714 struct sadata *sa = p->p_sa; 715 716 /* 717 * Defer saving the lwp's state because on some ports 718 * preemption can occur between generating an unblocked upcall 719 * and processing the upcall queue. 720 */ 721 if (sa->sa_flag & SA_FLAG_PREEMPT) 722 sa_upcall(l, SA_UPCALL_PREEMPTED | SA_UPCALL_DEFER_EVENT, 723 l, NULL, 0, NULL); 724 } 725 726 727 /* 728 * Set up the user-level stack and trapframe to do an upcall. 729 * 730 * NOTE: This routine WILL FREE "arg" in the case of failure! Callers 731 * should not touch the "arg" pointer once calling sa_upcall(). 732 */ 733 int 734 sa_upcall(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted, 735 size_t argsize, void *arg) 736 { 737 struct sadata_upcall *sau; 738 struct sadata *sa = l->l_proc->p_sa; 739 struct sadata_vp *vp = l->l_savp; 740 struct sastack *sast; 741 int error, f; 742 743 /* XXX prevent recursive upcalls if we sleep for memory */ 744 SA_LWP_STATE_LOCK(l, f); 745 sast = sa_getstack(sa); 746 SA_LWP_STATE_UNLOCK(l, f); 747 if (sast == NULL) { 748 return (ENOMEM); 749 } 750 DPRINTFN(9,("sa_upcall(%d.%d) using stack %p\n", 751 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp)); 752 753 SA_LWP_STATE_LOCK(l, f); 754 sau = sadata_upcall_alloc(1); 755 SA_LWP_STATE_UNLOCK(l, f); 756 error = sa_upcall0(l, type, event, interrupted, argsize, arg, sau); 757 if (error) { 758 sadata_upcall_free(sau); 759 sa_setstackfree(sast, sa); 760 return (error); 761 } 762 sau->sau_stack = sast->sast_stack; 763 764 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); 765 l->l_flag |= L_SA_UPCALL; 766 767 return (0); 768 } 769 770 static int 771 sa_upcall0(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted, 772 size_t argsize, void *arg, struct sadata_upcall *sau) 773 { 774 775 KDASSERT((event == NULL) || (event != interrupted)); 776 777 sau->sau_flags = 0; 778 sau->sau_type = type & SA_UPCALL_TYPE_MASK; 779 sau->sau_argsize = argsize; 780 sau->sau_arg = arg; 781 782 if (type & SA_UPCALL_DEFER_EVENT) { 783 sau->sau_event.ss_deferred.ss_lwp = event; 784 sau->sau_flags |= SAU_FLAG_DEFERRED_EVENT; 785 } else 786 sa_upcall_getstate(&sau->sau_event, event); 787 if (type & SA_UPCALL_DEFER_INTERRUPTED) { 788 sau->sau_interrupted.ss_deferred.ss_lwp = interrupted; 789 sau->sau_flags |= SAU_FLAG_DEFERRED_INTERRUPTED; 790 } else 791 sa_upcall_getstate(&sau->sau_interrupted, interrupted); 792 793 return (0); 794 } 795 796 797 static void 798 sa_upcall_getstate(union sau_state *ss, struct lwp *l) 799 { 800 801 if (l) { 802 l->l_flag |= L_SA_SWITCHING; 803 getucontext(l, &ss->ss_captured.ss_ctx); 804 l->l_flag &= ~L_SA_SWITCHING; 805 ss->ss_captured.ss_sa.sa_context = (ucontext_t *) 806 (intptr_t)((_UC_MACHINE_SP(&ss->ss_captured.ss_ctx) - 807 sizeof(ucontext_t)) 808 #ifdef _UC_UCONTEXT_ALIGN 809 & _UC_UCONTEXT_ALIGN 810 #endif 811 ); 812 ss->ss_captured.ss_sa.sa_id = l->l_lid; 813 ss->ss_captured.ss_sa.sa_cpu = l->l_savp->savp_id; 814 } else 815 ss->ss_captured.ss_sa.sa_context = NULL; 816 } 817 818 819 /* 820 * Detect double pagefaults and pagefaults on upcalls. 821 * - double pagefaults are detected by comparing the previous faultaddr 822 * against the current faultaddr 823 * - pagefaults on upcalls are detected by checking if the userspace 824 * thread is running on an upcall stack 825 */ 826 static __inline int 827 sa_pagefault(struct lwp *l, ucontext_t *l_ctx) 828 { 829 struct proc *p; 830 struct sadata *sa; 831 struct sadata_vp *vp; 832 struct sastack sast; 833 834 p = l->l_proc; 835 sa = p->p_sa; 836 vp = l->l_savp; 837 838 KDASSERT(vp->savp_lwp == l); 839 840 if (vp->savp_faultaddr == vp->savp_ofaultaddr) { 841 DPRINTFN(10,("sa_pagefault(%d.%d) double page fault\n", 842 p->p_pid, l->l_lid)); 843 return 1; 844 } 845 846 sast.sast_stack.ss_sp = (void *)(intptr_t)_UC_MACHINE_SP(l_ctx); 847 sast.sast_stack.ss_size = 1; 848 849 if (SPLAY_FIND(sasttree, &sa->sa_stackstree, &sast)) { 850 DPRINTFN(10,("sa_pagefault(%d.%d) upcall page fault\n", 851 p->p_pid, l->l_lid)); 852 return 1; 853 } 854 855 vp->savp_ofaultaddr = vp->savp_faultaddr; 856 return 0; 857 } 858 859 860 /* 861 * Called by tsleep(). Block current LWP and switch to another. 862 * 863 * WE ARE NOT ALLOWED TO SLEEP HERE! WE ARE CALLED FROM WITHIN 864 * TSLEEP() ITSELF! We are called with sched_lock held, and must 865 * hold it right through the mi_switch() call. 866 */ 867 void 868 sa_switch(struct lwp *l, int type) 869 { 870 struct proc *p = l->l_proc; 871 struct sadata_vp *vp = l->l_savp; 872 struct sadata_upcall *sau; 873 struct lwp *l2; 874 int error, s; 875 876 DPRINTFN(4,("sa_switch(%d.%d type %d VP %d)\n", p->p_pid, l->l_lid, 877 type, vp->savp_lwp ? vp->savp_lwp->l_lid : 0)); 878 879 SCHED_ASSERT_LOCKED(); 880 881 if (p->p_flag & P_WEXIT) { 882 mi_switch(l, NULL); 883 return; 884 } 885 886 if (l->l_flag & L_SA_YIELD) { 887 /* 888 * Case 0: we're blocking in sa_yield 889 */ 890 if (vp->savp_wokenq_head == NULL && p->p_userret == NULL) { 891 l->l_flag |= L_SA_IDLE; 892 mi_switch(l, NULL); 893 } else { 894 /* make us running again. */ 895 unsleep(l); 896 l->l_stat = LSONPROC; 897 l->l_proc->p_nrlwps++; 898 s = splsched(); 899 SCHED_UNLOCK(s); 900 } 901 return; 902 } else if (vp->savp_lwp == l) { 903 /* 904 * Case 1: we're blocking for the first time; generate 905 * a SA_BLOCKED upcall and allocate resources for the 906 * UNBLOCKED upcall. 907 */ 908 909 /* 910 * The process of allocating a new LWP could cause 911 * sleeps. We're called from inside sleep, so that 912 * would be Bad. Therefore, we must use a cached new 913 * LWP. The first thing that this new LWP must do is 914 * allocate another LWP for the cache. */ 915 l2 = sa_getcachelwp(vp); 916 if (l2 == NULL) { 917 /* XXXSMP */ 918 /* No upcall for you! */ 919 /* XXX The consequences of this are more subtle and 920 * XXX the recovery from this situation deserves 921 * XXX more thought. 922 */ 923 924 /* XXXUPSXXX Should only happen with concurrency > 1 */ 925 #ifdef DIAGNOSTIC 926 printf("sa_switch(%d.%d): no cached LWP for upcall.\n", 927 p->p_pid, l->l_lid); 928 #endif 929 mi_switch(l, NULL); 930 return; 931 } 932 933 /* 934 * XXX We need to allocate the sadata_upcall structure here, 935 * XXX since we can't sleep while waiting for memory inside 936 * XXX sa_upcall(). It would be nice if we could safely 937 * XXX allocate the sadata_upcall structure on the stack, here. 938 */ 939 sau = sadata_upcall_alloc(0); 940 if (sau == NULL) { 941 #ifdef DIAGNOSTIC 942 printf("sa_switch(%d.%d): couldn't allocate upcall data.\n", 943 p->p_pid, l->l_lid); 944 #endif 945 sa_putcachelwp(p, l2); /* PHOLD from sa_getcachelwp */ 946 mi_switch(l, NULL); 947 return; 948 } 949 950 cpu_setfunc(l2, sa_switchcall, sau); 951 error = sa_upcall0(l2, SA_UPCALL_BLOCKED, l, NULL, 0, NULL, 952 sau); 953 if (error) { 954 #ifdef DIAGNOSTIC 955 printf("sa_switch(%d.%d): Error %d from sa_upcall()\n", 956 p->p_pid, l->l_lid, error); 957 #endif 958 sa_putcachelwp(p, l2); /* PHOLD from sa_getcachelwp */ 959 mi_switch(l, NULL); 960 return; 961 } 962 963 /* 964 * Perform the double/upcall pagefault check. 965 * We do this only here since we need l's ucontext to 966 * get l's userspace stack. sa_upcall0 above has saved 967 * it for us. 968 * The L_SA_PAGEFAULT flag is set in the MD 969 * pagefault code to indicate a pagefault. The MD 970 * pagefault code also saves the faultaddr for us. 971 */ 972 if ((l->l_flag & L_SA_PAGEFAULT) && sa_pagefault(l, 973 &sau->sau_event.ss_captured.ss_ctx) != 0) { 974 sadata_upcall_free(sau); 975 sa_putcachelwp(p, l2); /* PHOLD from sa_getcachelwp */ 976 mi_switch(l, NULL); 977 DPRINTFN(10,("sa_switch(%d.%d) page fault resolved\n", 978 p->p_pid, l->l_lid)); 979 if (vp->savp_faultaddr == vp->savp_ofaultaddr) 980 vp->savp_ofaultaddr = -1; 981 return; 982 } 983 984 DPRINTFN(8,("sa_switch(%d.%d) blocked upcall %d\n", 985 p->p_pid, l->l_lid, l2->l_lid)); 986 987 l->l_flag |= L_SA_BLOCKING; 988 l2->l_priority = l2->l_usrpri; 989 vp->savp_blocker = l; 990 vp->savp_lwp = l2; 991 setrunnable(l2); 992 PRELE(l2); /* Remove the artificial hold-count */ 993 994 KDASSERT(l2 != l); 995 } else if (vp->savp_lwp != NULL) { 996 /* 997 * Case 2: We've been woken up while another LWP was 998 * on the VP, but we're going back to sleep without 999 * having returned to userland and delivering the 1000 * SA_UNBLOCKED upcall (select and poll cause this 1001 * kind of behavior a lot). We just switch back to the 1002 * LWP that had been running and let it have another 1003 * go. If the LWP on the VP was idling, don't make it 1004 * run again, though. 1005 */ 1006 if (vp->savp_lwp->l_flag & L_SA_YIELD) 1007 l2 = NULL; 1008 else { 1009 l2 = vp->savp_lwp; /* XXXUPSXXX Unfair advantage for l2 ? */ 1010 if((l2->l_stat != LSRUN) || ((l2->l_flag & L_INMEM) == 0)) 1011 l2 = NULL; 1012 } 1013 } else { 1014 /* NOTREACHED */ 1015 panic("sa_vp empty"); 1016 } 1017 1018 DPRINTFN(4,("sa_switch(%d.%d) switching to LWP %d.\n", 1019 p->p_pid, l->l_lid, l2 ? l2->l_lid : 0)); 1020 mi_switch(l, l2); 1021 1022 DPRINTFN(4,("sa_switch(%d.%d flag %x) returned.\n", 1023 p->p_pid, l->l_lid, l->l_flag)); 1024 KDASSERT(l->l_wchan == 0); 1025 1026 SCHED_ASSERT_UNLOCKED(); 1027 } 1028 1029 static void 1030 sa_switchcall(void *arg) 1031 { 1032 struct lwp *l, *l2; 1033 struct proc *p; 1034 struct sadata_vp *vp; 1035 struct sadata_upcall *sau; 1036 struct sastack *sast; 1037 int s; 1038 1039 l2 = curlwp; 1040 p = l2->l_proc; 1041 vp = l2->l_savp; 1042 1043 if (p->p_flag & P_WEXIT) 1044 lwp_exit(l2); 1045 1046 KDASSERT(vp->savp_lwp == l2); 1047 sau = arg; 1048 1049 DPRINTFN(6,("sa_switchcall(%d.%d)\n", p->p_pid, l2->l_lid)); 1050 1051 l2->l_flag &= ~L_SA; 1052 if (LIST_EMPTY(&vp->savp_lwpcache)) { 1053 /* Allocate the next cache LWP */ 1054 DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n", 1055 p->p_pid, l2->l_lid)); 1056 sa_newcachelwp(l2); 1057 } 1058 if (sau) { 1059 l = vp->savp_blocker; 1060 sast = sa_getstack(p->p_sa); 1061 if (sast) { 1062 sau->sau_stack = sast->sast_stack; 1063 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); 1064 l2->l_flag |= L_SA_UPCALL; 1065 } else { 1066 #ifdef DIAGNOSTIC 1067 printf("sa_switchcall(%d.%d flag %x): Not enough stacks.\n", 1068 p->p_pid, l->l_lid, l->l_flag); 1069 #endif 1070 sadata_upcall_free(sau); 1071 PHOLD(l2); 1072 SCHED_LOCK(s); 1073 sa_putcachelwp(p, l2); /* sets L_SA */ 1074 vp->savp_lwp = l; 1075 mi_switch(l2, NULL); 1076 /* mostly NOTREACHED */ 1077 SCHED_ASSERT_UNLOCKED(); 1078 splx(s); 1079 } 1080 } 1081 l2->l_flag |= L_SA; 1082 1083 upcallret(l2); 1084 } 1085 1086 static int 1087 sa_newcachelwp(struct lwp *l) 1088 { 1089 struct proc *p; 1090 struct lwp *l2; 1091 vaddr_t uaddr; 1092 boolean_t inmem; 1093 int s; 1094 1095 p = l->l_proc; 1096 if (p->p_flag & P_WEXIT) 1097 return (0); 1098 1099 inmem = uvm_uarea_alloc(&uaddr); 1100 if (__predict_false(uaddr == 0)) { 1101 return (ENOMEM); 1102 } else { 1103 newlwp(l, p, uaddr, inmem, 0, NULL, 0, child_return, 0, &l2); 1104 /* We don't want this LWP on the process's main LWP list, but 1105 * newlwp helpfully puts it there. Unclear if newlwp should 1106 * be tweaked. 1107 */ 1108 PHOLD(l2); 1109 SCHED_LOCK(s); 1110 l2->l_savp = l->l_savp; 1111 sa_putcachelwp(p, l2); 1112 SCHED_UNLOCK(s); 1113 } 1114 1115 return (0); 1116 } 1117 1118 /* 1119 * Take a normal process LWP and place it in the SA cache. 1120 * LWP must not be running! 1121 */ 1122 void 1123 sa_putcachelwp(struct proc *p, struct lwp *l) 1124 { 1125 struct sadata_vp *vp; 1126 1127 SCHED_ASSERT_LOCKED(); 1128 1129 vp = l->l_savp; 1130 1131 LIST_REMOVE(l, l_sibling); 1132 p->p_nlwps--; 1133 l->l_stat = LSSUSPENDED; 1134 l->l_flag |= (L_DETACHED | L_SA); 1135 /* XXX lock sadata */ 1136 DPRINTFN(5,("sa_putcachelwp(%d.%d) Adding LWP %d to cache\n", 1137 p->p_pid, curlwp->l_lid, l->l_lid)); 1138 LIST_INSERT_HEAD(&vp->savp_lwpcache, l, l_sibling); 1139 vp->savp_ncached++; 1140 /* XXX unlock */ 1141 } 1142 1143 /* 1144 * Fetch a LWP from the cache. 1145 */ 1146 struct lwp * 1147 sa_getcachelwp(struct sadata_vp *vp) 1148 { 1149 struct lwp *l; 1150 struct proc *p; 1151 1152 SCHED_ASSERT_LOCKED(); 1153 1154 l = NULL; 1155 /* XXX lock sadata */ 1156 if (vp->savp_ncached > 0) { 1157 vp->savp_ncached--; 1158 l = LIST_FIRST(&vp->savp_lwpcache); 1159 LIST_REMOVE(l, l_sibling); 1160 p = l->l_proc; 1161 LIST_INSERT_HEAD(&p->p_lwps, l, l_sibling); 1162 p->p_nlwps++; 1163 DPRINTFN(5,("sa_getcachelwp(%d.%d) Got LWP %d from cache.\n", 1164 p->p_pid, curlwp->l_lid, l->l_lid)); 1165 } 1166 /* XXX unlock */ 1167 return l; 1168 } 1169 1170 1171 void 1172 sa_unblock_userret(struct lwp *l) 1173 { 1174 struct proc *p; 1175 struct lwp *l2; 1176 struct sadata *sa; 1177 struct sadata_vp *vp; 1178 struct sadata_upcall *sau; 1179 struct sastack *sast; 1180 int f, s; 1181 1182 p = l->l_proc; 1183 sa = p->p_sa; 1184 vp = l->l_savp; 1185 1186 if (p->p_flag & P_WEXIT) 1187 return; 1188 1189 SCHED_ASSERT_UNLOCKED(); 1190 1191 KERNEL_PROC_LOCK(l); 1192 SA_LWP_STATE_LOCK(l, f); 1193 1194 DPRINTFN(7,("sa_unblock_userret(%d.%d %x) \n", p->p_pid, l->l_lid, 1195 l->l_flag)); 1196 1197 sa_setwoken(l); 1198 /* maybe NOTREACHED */ 1199 1200 SCHED_LOCK(s); 1201 if (l != vp->savp_lwp) { 1202 /* Invoke an "unblocked" upcall */ 1203 DPRINTFN(8,("sa_unblock_userret(%d.%d) unblocking\n", 1204 p->p_pid, l->l_lid)); 1205 1206 l2 = sa_vp_repossess(l); 1207 1208 SCHED_UNLOCK(s); 1209 1210 if (l2 == NULL) 1211 lwp_exit(l); 1212 1213 sast = sa_getstack(sa); 1214 if (p->p_flag & P_WEXIT) 1215 lwp_exit(l); 1216 1217 sau = sadata_upcall_alloc(1); 1218 sau->sau_arg = NULL; 1219 if (p->p_flag & P_WEXIT) { 1220 sadata_upcall_free(sau); 1221 lwp_exit(l); 1222 } 1223 1224 PHOLD(l2); 1225 1226 KDASSERT(sast != NULL); 1227 DPRINTFN(9,("sa_unblock_userret(%d.%d) using stack %p\n", 1228 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp)); 1229 1230 /* 1231 * Defer saving the event lwp's state because a 1232 * PREEMPT upcall could be on the queue already. 1233 */ 1234 if (sa_upcall0(l, SA_UPCALL_UNBLOCKED | SA_UPCALL_DEFER_EVENT, 1235 l, l2, 0, NULL, sau) != 0) { 1236 /* 1237 * We were supposed to deliver an UNBLOCKED 1238 * upcall, but don't have resources to do so. 1239 */ 1240 #ifdef DIAGNOSTIC 1241 printf("sa_unblock_userret: out of upcall resources" 1242 " for %d.%d\n", p->p_pid, l->l_lid); 1243 #endif 1244 sigexit(l, SIGABRT); 1245 /* NOTREACHED */ 1246 } 1247 sau->sau_stack = sast->sast_stack; 1248 1249 SCHED_LOCK(s); 1250 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); 1251 l->l_flag |= L_SA_UPCALL; 1252 l->l_flag &= ~L_SA_BLOCKING; 1253 sa_putcachelwp(p, l2); 1254 } 1255 SCHED_UNLOCK(s); 1256 1257 SA_LWP_STATE_UNLOCK(l, f); 1258 KERNEL_PROC_UNLOCK(l); 1259 } 1260 1261 void 1262 sa_upcall_userret(struct lwp *l) 1263 { 1264 struct lwp *l2; 1265 struct proc *p; 1266 struct sadata *sa; 1267 struct sadata_vp *vp; 1268 struct sadata_upcall *sau; 1269 struct sastack *sast; 1270 int f, s; 1271 1272 p = l->l_proc; 1273 sa = p->p_sa; 1274 vp = l->l_savp; 1275 1276 SCHED_ASSERT_UNLOCKED(); 1277 1278 KERNEL_PROC_LOCK(l); 1279 SA_LWP_STATE_LOCK(l, f); 1280 1281 DPRINTFN(7,("sa_upcall_userret(%d.%d %x) \n", p->p_pid, l->l_lid, 1282 l->l_flag)); 1283 1284 KDASSERT((l->l_flag & L_SA_BLOCKING) == 0); 1285 1286 sast = NULL; 1287 if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL) 1288 sast = sa_getstack(sa); 1289 SCHED_LOCK(s); 1290 if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL && 1291 sast != NULL) { 1292 /* Invoke an "unblocked" upcall */ 1293 l2 = vp->savp_wokenq_head; 1294 vp->savp_wokenq_head = l2->l_forw; 1295 1296 DPRINTFN(9,("sa_upcall_userret(%d.%d) using stack %p\n", 1297 l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp)); 1298 1299 SCHED_UNLOCK(s); 1300 1301 if (p->p_flag & P_WEXIT) 1302 lwp_exit(l); 1303 1304 DPRINTFN(8,("sa_upcall_userret(%d.%d) unblocking %d\n", 1305 p->p_pid, l->l_lid, l2->l_lid)); 1306 1307 sau = sadata_upcall_alloc(1); 1308 sau->sau_arg = NULL; 1309 if (p->p_flag & P_WEXIT) { 1310 sadata_upcall_free(sau); 1311 lwp_exit(l); 1312 } 1313 1314 if (sa_upcall0(l, SA_UPCALL_UNBLOCKED, l2, l, 0, NULL, 1315 sau) != 0) { 1316 /* 1317 * We were supposed to deliver an UNBLOCKED 1318 * upcall, but don't have resources to do so. 1319 */ 1320 #ifdef DIAGNOSTIC 1321 printf("sa_upcall_userret: out of upcall resources" 1322 " for %d.%d\n", p->p_pid, l->l_lid); 1323 #endif 1324 sigexit(l, SIGABRT); 1325 /* NOTREACHED */ 1326 } 1327 sau->sau_stack = sast->sast_stack; 1328 1329 SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); 1330 1331 l2->l_flag &= ~L_SA_BLOCKING; 1332 SCHED_LOCK(s); 1333 sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */ 1334 SCHED_UNLOCK(s); 1335 } else { 1336 SCHED_UNLOCK(s); 1337 if (sast) 1338 sa_setstackfree(sast, sa); 1339 } 1340 1341 KDASSERT(vp->savp_lwp == l); 1342 1343 while (!SIMPLEQ_EMPTY(&vp->savp_upcalls)) 1344 sa_makeupcalls(l); 1345 1346 if (vp->savp_wokenq_head == NULL) 1347 l->l_flag &= ~L_SA_UPCALL; 1348 1349 SA_LWP_STATE_UNLOCK(l, f); 1350 KERNEL_PROC_UNLOCK(l); 1351 return; 1352 } 1353 1354 static __inline void 1355 sa_makeupcalls(struct lwp *l) 1356 { 1357 struct lwp *l2, *eventq; 1358 struct proc *p; 1359 struct sadata *sa; 1360 struct sadata_vp *vp; 1361 struct sa_t **sapp, *sap; 1362 struct sa_t self_sa; 1363 struct sa_t *sas[3], *sasp; 1364 struct sadata_upcall *sau; 1365 union sau_state e_ss; 1366 void *stack, *ap; 1367 ucontext_t u, *up; 1368 int i, nint, nevents, s, type; 1369 1370 p = l->l_proc; 1371 sa = p->p_sa; 1372 vp = l->l_savp; 1373 1374 sau = SIMPLEQ_FIRST(&vp->savp_upcalls); 1375 SIMPLEQ_REMOVE_HEAD(&vp->savp_upcalls, sau_next); 1376 1377 if (sau->sau_flags & SAU_FLAG_DEFERRED_EVENT) 1378 sa_upcall_getstate(&sau->sau_event, 1379 sau->sau_event.ss_deferred.ss_lwp); 1380 if (sau->sau_flags & SAU_FLAG_DEFERRED_INTERRUPTED) 1381 sa_upcall_getstate(&sau->sau_interrupted, 1382 sau->sau_interrupted.ss_deferred.ss_lwp); 1383 1384 stack = (void *) 1385 (((uintptr_t)sau->sau_stack.ss_sp + sau->sau_stack.ss_size) 1386 & ~ALIGNBYTES); 1387 1388 self_sa.sa_id = l->l_lid; 1389 self_sa.sa_cpu = vp->savp_id; 1390 sas[0] = &self_sa; 1391 nevents = 0; 1392 nint = 0; 1393 if (sau->sau_event.ss_captured.ss_sa.sa_context != NULL) { 1394 if (copyout(&sau->sau_event.ss_captured.ss_ctx, 1395 sau->sau_event.ss_captured.ss_sa.sa_context, 1396 sizeof(ucontext_t)) != 0) { 1397 #ifdef DIAGNOSTIC 1398 printf("sa_makeupcalls(%d.%d): couldn't copyout" 1399 " context of event LWP %d\n", 1400 p->p_pid, l->l_lid, sau->sau_event.ss_captured.ss_sa.sa_id); 1401 #endif 1402 sigexit(l, SIGILL); 1403 /* NOTREACHED */ 1404 } 1405 sas[1] = &sau->sau_event.ss_captured.ss_sa; 1406 nevents = 1; 1407 } 1408 if (sau->sau_interrupted.ss_captured.ss_sa.sa_context != NULL) { 1409 KDASSERT(sau->sau_interrupted.ss_captured.ss_sa.sa_context != 1410 sau->sau_event.ss_captured.ss_sa.sa_context); 1411 if (copyout(&sau->sau_interrupted.ss_captured.ss_ctx, 1412 sau->sau_interrupted.ss_captured.ss_sa.sa_context, 1413 sizeof(ucontext_t)) != 0) { 1414 #ifdef DIAGNOSTIC 1415 printf("sa_makeupcalls(%d.%d): couldn't copyout" 1416 " context of interrupted LWP %d\n", 1417 p->p_pid, l->l_lid, sau->sau_interrupted.ss_captured.ss_sa.sa_id); 1418 #endif 1419 sigexit(l, SIGILL); 1420 /* NOTREACHED */ 1421 } 1422 sas[2] = &sau->sau_interrupted.ss_captured.ss_sa; 1423 nint = 1; 1424 } 1425 eventq = NULL; 1426 if (sau->sau_type == SA_UPCALL_UNBLOCKED) { 1427 SCHED_LOCK(s); 1428 eventq = vp->savp_wokenq_head; 1429 vp->savp_wokenq_head = NULL; 1430 SCHED_UNLOCK(s); 1431 l2 = eventq; 1432 while (l2 != NULL) { 1433 nevents++; 1434 l2 = l2->l_forw; 1435 } 1436 } 1437 1438 /* Copy out the activation's ucontext */ 1439 u.uc_stack = sau->sau_stack; 1440 u.uc_flags = _UC_STACK; 1441 up = stack; 1442 up--; 1443 if (copyout(&u, up, sizeof(ucontext_t)) != 0) { 1444 sadata_upcall_free(sau); 1445 #ifdef DIAGNOSTIC 1446 printf("sa_makeupcalls: couldn't copyout activation" 1447 " ucontext for %d.%d to %p\n", l->l_proc->p_pid, l->l_lid, 1448 up); 1449 #endif 1450 sigexit(l, SIGILL); 1451 /* NOTREACHED */ 1452 } 1453 sas[0]->sa_context = up; 1454 1455 /* Next, copy out the sa_t's and pointers to them. */ 1456 sap = (struct sa_t *) up; 1457 sapp = (struct sa_t **) (sap - (1 + nevents + nint)); 1458 KDASSERT(nint <= 1); 1459 for (i = nevents + nint; i >= 0; i--) { 1460 sap--; 1461 sapp--; 1462 if (i == 1 + nevents) /* interrupted sa */ 1463 sasp = sas[2]; 1464 else if (i <= 1) /* self_sa and event sa */ 1465 sasp = sas[i]; 1466 else { /* extra sas */ 1467 KDASSERT(sau->sau_type == SA_UPCALL_UNBLOCKED); 1468 KDASSERT(eventq != NULL); 1469 l2 = eventq; 1470 KDASSERT(l2 != NULL); 1471 eventq = l2->l_forw; 1472 DPRINTFN(8,("sa_makeupcalls(%d.%d) unblocking extra %d\n", 1473 p->p_pid, l->l_lid, l2->l_lid)); 1474 sa_upcall_getstate(&e_ss, l2); 1475 SCHED_LOCK(s); 1476 l2->l_flag &= ~L_SA_BLOCKING; 1477 sa_putcachelwp(p, l2); /* PHOLD from sa_setwoken */ 1478 SCHED_UNLOCK(s); 1479 if (copyout(&e_ss.ss_captured.ss_ctx, 1480 e_ss.ss_captured.ss_sa.sa_context, 1481 sizeof(ucontext_t)) != 0) { 1482 #ifdef DIAGNOSTIC 1483 printf("sa_makeupcalls(%d.%d): couldn't copyout" 1484 " context of event LWP %d\n", 1485 p->p_pid, l->l_lid, e_ss.ss_captured.ss_sa.sa_id); 1486 #endif 1487 sigexit(l, SIGILL); 1488 /* NOTREACHED */ 1489 } 1490 sasp = &e_ss.ss_captured.ss_sa; 1491 } 1492 if ((copyout(sasp, sap, sizeof(struct sa_t)) != 0) || 1493 (copyout(&sap, sapp, sizeof(struct sa_t *)) != 0)) { 1494 /* Copying onto the stack didn't work. Die. */ 1495 sadata_upcall_free(sau); 1496 #ifdef DIAGNOSTIC 1497 printf("sa_makeupcalls: couldn't copyout sa_t " 1498 "%d for %d.%d\n", i, p->p_pid, l->l_lid); 1499 #endif 1500 sigexit(l, SIGILL); 1501 /* NOTREACHED */ 1502 } 1503 } 1504 KDASSERT(eventq == NULL); 1505 1506 /* Copy out the arg, if any */ 1507 /* xxx assume alignment works out; everything so far has been 1508 * a structure, so... 1509 */ 1510 if (sau->sau_arg) { 1511 ap = (char *)sapp - sau->sau_argsize; 1512 stack = ap; 1513 if (copyout(sau->sau_arg, ap, sau->sau_argsize) != 0) { 1514 /* Copying onto the stack didn't work. Die. */ 1515 sadata_upcall_free(sau); 1516 #ifdef DIAGNOSTIC 1517 printf("sa_makeupcalls(%d.%d): couldn't copyout" 1518 " sadata_upcall arg %p size %ld to %p \n", 1519 p->p_pid, l->l_lid, 1520 sau->sau_arg, (long) sau->sau_argsize, ap); 1521 #endif 1522 sigexit(l, SIGILL); 1523 /* NOTREACHED */ 1524 } 1525 } else { 1526 ap = 0; 1527 stack = sapp; 1528 } 1529 1530 type = sau->sau_type; 1531 1532 sadata_upcall_free(sau); 1533 1534 DPRINTFN(7,("sa_makeupcalls(%d.%d): type %d\n", p->p_pid, 1535 l->l_lid, type)); 1536 1537 cpu_upcall(l, type, nevents, nint, sapp, ap, stack, sa->sa_upcall); 1538 1539 l->l_flag &= ~L_SA_YIELD; 1540 } 1541 1542 static void 1543 sa_setwoken(struct lwp *l) 1544 { 1545 struct lwp *l2, *vp_lwp; 1546 struct proc *p; 1547 struct sadata *sa; 1548 struct sadata_vp *vp; 1549 int s; 1550 1551 SCHED_LOCK(s); 1552 1553 if ((l->l_flag & L_SA_BLOCKING) == 0) { 1554 SCHED_UNLOCK(s); 1555 return; 1556 } 1557 1558 p = l->l_proc; 1559 sa = p->p_sa; 1560 vp = l->l_savp; 1561 vp_lwp = vp->savp_lwp; 1562 l2 = NULL; 1563 1564 KDASSERT(vp_lwp != NULL); 1565 DPRINTFN(3,("sa_setwoken(%d.%d) woken, flags %x, vp %d\n", 1566 l->l_proc->p_pid, l->l_lid, l->l_flag, 1567 vp_lwp->l_lid)); 1568 1569 #if notyet 1570 if (vp_lwp->l_flag & L_SA_IDLE) { 1571 KDASSERT((vp_lwp->l_flag & L_SA_UPCALL) == 0); 1572 KDASSERT(vp->savp_wokenq_head == NULL); 1573 DPRINTFN(3,("sa_setwoken(%d.%d) repossess: idle vp_lwp %d state %d\n", 1574 l->l_proc->p_pid, l->l_lid, 1575 vp_lwp->l_lid, vp_lwp->l_stat)); 1576 vp_lwp->l_flag &= ~L_SA_IDLE; 1577 SCHED_UNLOCK(s); 1578 return; 1579 } 1580 #endif 1581 1582 DPRINTFN(3,("sa_setwoken(%d.%d) put on wokenq: vp_lwp %d state %d\n", 1583 l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid, 1584 vp_lwp->l_stat)); 1585 1586 PHOLD(l); 1587 if (vp->savp_wokenq_head == NULL) 1588 vp->savp_wokenq_head = l; 1589 else 1590 *vp->savp_wokenq_tailp = l; 1591 *(vp->savp_wokenq_tailp = &l->l_forw) = NULL; 1592 1593 switch (vp_lwp->l_stat) { 1594 case LSONPROC: 1595 if (vp_lwp->l_flag & L_SA_UPCALL) 1596 break; 1597 vp_lwp->l_flag |= L_SA_UPCALL; 1598 if (vp_lwp->l_flag & L_SA_YIELD) 1599 break; 1600 /* XXX IPI vp_lwp->l_cpu */ 1601 break; 1602 case LSSLEEP: 1603 if (vp_lwp->l_flag & L_SA_IDLE) { 1604 vp_lwp->l_flag &= ~L_SA_IDLE; 1605 vp_lwp->l_flag |= L_SA_UPCALL; 1606 setrunnable(vp_lwp); 1607 break; 1608 } 1609 vp_lwp->l_flag |= L_SA_UPCALL; 1610 break; 1611 case LSSUSPENDED: 1612 #ifdef DIAGNOSTIC 1613 printf("sa_setwoken(%d.%d) vp lwp %d LSSUSPENDED\n", 1614 l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid); 1615 #endif 1616 break; 1617 case LSSTOP: 1618 vp_lwp->l_flag |= L_SA_UPCALL; 1619 break; 1620 case LSRUN: 1621 if (vp_lwp->l_flag & L_SA_UPCALL) 1622 break; 1623 vp_lwp->l_flag |= L_SA_UPCALL; 1624 if (vp_lwp->l_flag & L_SA_YIELD) 1625 break; 1626 if (vp_lwp->l_slptime > 1) { 1627 void updatepri(struct lwp *); 1628 updatepri(vp_lwp); 1629 } 1630 vp_lwp->l_slptime = 0; 1631 if (vp_lwp->l_flag & L_INMEM) { 1632 if (vp_lwp->l_cpu == curcpu()) 1633 l2 = vp_lwp; 1634 else 1635 need_resched(vp_lwp->l_cpu); 1636 } else 1637 sched_wakeup(&proc0); 1638 break; 1639 default: 1640 panic("sa_vp LWP not sleeping/onproc/runnable"); 1641 } 1642 1643 l->l_stat = LSSUSPENDED; 1644 p->p_nrlwps--; 1645 mi_switch(l, l2); 1646 /* maybe NOTREACHED */ 1647 SCHED_ASSERT_UNLOCKED(); 1648 splx(s); 1649 if (p->p_flag & P_WEXIT) 1650 lwp_exit(l); 1651 } 1652 1653 static struct lwp * 1654 sa_vp_repossess(struct lwp *l) 1655 { 1656 struct lwp *l2; 1657 struct proc *p = l->l_proc; 1658 struct sadata_vp *vp = l->l_savp; 1659 1660 SCHED_ASSERT_LOCKED(); 1661 1662 /* 1663 * Put ourselves on the virtual processor and note that the 1664 * previous occupant of that position was interrupted. 1665 */ 1666 l2 = vp->savp_lwp; 1667 vp->savp_lwp = l; 1668 if (l2->l_flag & L_SA_YIELD) 1669 l2->l_flag &= ~(L_SA_YIELD|L_SA_IDLE); 1670 1671 DPRINTFN(1,("sa_vp_repossess(%d.%d) vp lwp %d state %d\n", 1672 p->p_pid, l->l_lid, l2->l_lid, l2->l_stat)); 1673 1674 KDASSERT(l2 != l); 1675 if (l2) { 1676 switch (l2->l_stat) { 1677 case LSRUN: 1678 remrunqueue(l2); 1679 p->p_nrlwps--; 1680 break; 1681 case LSSLEEP: 1682 unsleep(l2); 1683 l2->l_flag &= ~L_SINTR; 1684 break; 1685 case LSSUSPENDED: 1686 #ifdef DIAGNOSTIC 1687 printf("sa_vp_repossess(%d.%d) vp lwp %d LSSUSPENDED\n", 1688 l->l_proc->p_pid, l->l_lid, l2->l_lid); 1689 #endif 1690 break; 1691 #ifdef DIAGNOSTIC 1692 default: 1693 panic("SA VP %d.%d is in state %d, not running" 1694 " or sleeping\n", p->p_pid, l2->l_lid, 1695 l2->l_stat); 1696 #endif 1697 } 1698 l2->l_stat = LSSUSPENDED; 1699 } 1700 return l2; 1701 } 1702 1703 1704 1705 #ifdef DEBUG 1706 int debug_print_sa(struct proc *); 1707 int debug_print_lwp(struct lwp *); 1708 int debug_print_proc(int); 1709 1710 int 1711 debug_print_proc(int pid) 1712 { 1713 struct proc *p; 1714 1715 p = pfind(pid); 1716 if (p == NULL) 1717 printf("No process %d\n", pid); 1718 else 1719 debug_print_sa(p); 1720 1721 return 0; 1722 } 1723 1724 int 1725 debug_print_sa(struct proc *p) 1726 { 1727 struct lwp *l; 1728 struct sadata *sa; 1729 struct sadata_vp *vp; 1730 1731 printf("Process %d (%s), state %d, address %p, flags %x\n", 1732 p->p_pid, p->p_comm, p->p_stat, p, p->p_flag); 1733 printf("LWPs: %d (%d running, %d zombies)\n", 1734 p->p_nlwps, p->p_nrlwps, p->p_nzlwps); 1735 LIST_FOREACH(l, &p->p_lwps, l_sibling) 1736 debug_print_lwp(l); 1737 sa = p->p_sa; 1738 if (sa) { 1739 SLIST_FOREACH(vp, &sa->sa_vps, savp_next) { 1740 if (vp->savp_lwp) 1741 printf("SA VP: %d %s\n", vp->savp_lwp->l_lid, 1742 vp->savp_lwp->l_flag & L_SA_YIELD ? 1743 (vp->savp_lwp->l_flag & L_SA_IDLE ? 1744 "idle" : "yielding") : ""); 1745 printf("SAs: %d cached LWPs\n", vp->savp_ncached); 1746 LIST_FOREACH(l, &vp->savp_lwpcache, l_sibling) 1747 debug_print_lwp(l); 1748 } 1749 } 1750 1751 return 0; 1752 } 1753 1754 int 1755 debug_print_lwp(struct lwp *l) 1756 { 1757 1758 printf("LWP %d address %p ", l->l_lid, l); 1759 printf("state %d flags %x ", l->l_stat, l->l_flag); 1760 if (l->l_wchan) 1761 printf("wait %p %s", l->l_wchan, l->l_wmesg); 1762 printf("\n"); 1763 1764 return 0; 1765 } 1766 1767 #endif
Cache object: 4ec157ae543607f3015ba3a9240ae929
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]