Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_kse.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD: releng/6.0/sys/kern/kern_kse.c 151012 2005-10-06 18:24:24Z delphij $"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/imgact.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/proc.h> 39 #include <sys/ptrace.h> 40 #include <sys/smp.h> 41 #include <sys/syscallsubr.h> 42 #include <sys/sysproto.h> 43 #include <sys/sched.h> 44 #include <sys/signalvar.h> 45 #include <sys/sleepqueue.h> 46 #include <sys/kse.h> 47 #include <sys/ktr.h> 48 #include <vm/uma.h> 49 50 /* 51 * KSEGRP related storage. 52 */ 53 static uma_zone_t upcall_zone; 54 55 /* DEBUG ONLY */ 56 extern int virtual_cpu; 57 extern int thread_debug; 58 59 extern int max_threads_per_proc; 60 extern int max_groups_per_proc; 61 extern int max_threads_hits; 62 extern struct mtx kse_zombie_lock; 63 64 65 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 66 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 67 68 static int thread_update_usr_ticks(struct thread *td); 69 static void thread_alloc_spare(struct thread *td); 70 71 struct kse_upcall * 72 upcall_alloc(void) 73 { 74 struct kse_upcall *ku; 75 76 ku = uma_zalloc(upcall_zone, M_WAITOK | M_ZERO); 77 return (ku); 78 } 79 80 void 81 upcall_free(struct kse_upcall *ku) 82 { 83 84 uma_zfree(upcall_zone, ku); 85 } 86 87 void 88 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 89 { 90 91 mtx_assert(&sched_lock, MA_OWNED); 92 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 93 ku->ku_ksegrp = kg; 94 kg->kg_numupcalls++; 95 } 96 97 void 98 upcall_unlink(struct kse_upcall *ku) 99 { 100 struct ksegrp *kg = ku->ku_ksegrp; 101 102 mtx_assert(&sched_lock, MA_OWNED); 103 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 104 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 105 kg->kg_numupcalls--; 106 upcall_stash(ku); 107 } 108 109 void 110 upcall_remove(struct thread *td) 111 { 112 113 mtx_assert(&sched_lock, MA_OWNED); 114 if (td->td_upcall != NULL) { 115 td->td_upcall->ku_owner = NULL; 116 upcall_unlink(td->td_upcall); 117 td->td_upcall = NULL; 118 } 119 } 120 121 #ifndef _SYS_SYSPROTO_H_ 122 struct kse_switchin_args { 123 struct kse_thr_mailbox *tmbx; 124 int flags; 125 }; 126 #endif 127 128 int 129 kse_switchin(struct thread *td, struct kse_switchin_args *uap) 130 { 131 struct kse_thr_mailbox tmbx; 132 struct kse_upcall *ku; 133 int error; 134 135 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 136 return (EINVAL); 137 error = (uap->tmbx == NULL) ? EINVAL : 0; 138 if (!error) 139 error = copyin(uap->tmbx, &tmbx, sizeof(tmbx)); 140 if (!error && (uap->flags & KSE_SWITCHIN_SETTMBX)) 141 error = (suword(&ku->ku_mailbox->km_curthread, 142 (long)uap->tmbx) != 0 ? EINVAL : 0); 143 if (!error) 144 error = set_mcontext(td, &tmbx.tm_context.uc_mcontext); 145 if (!error) { 146 suword32(&uap->tmbx->tm_lwp, td->td_tid); 147 if (uap->flags & KSE_SWITCHIN_SETTMBX) { 148 td->td_mailbox = uap->tmbx; 149 td->td_pflags |= TDP_CAN_UNBIND; 150 } 151 if (td->td_proc->p_flag & P_TRACED) { 152 if (tmbx.tm_dflags & TMDF_SSTEP) 153 ptrace_single_step(td); 154 else 155 ptrace_clear_single_step(td); 156 if (tmbx.tm_dflags & TMDF_SUSPEND) { 157 mtx_lock_spin(&sched_lock); 158 /* fuword can block, check again */ 159 if (td->td_upcall) 160 ku->ku_flags |= KUF_DOUPCALL; 161 mtx_unlock_spin(&sched_lock); 162 } 163 } 164 } 165 return ((error == 0) ? EJUSTRETURN : error); 166 } 167 168 /* 169 struct kse_thr_interrupt_args { 170 struct kse_thr_mailbox * tmbx; 171 int cmd; 172 long data; 173 }; 174 */ 175 int 176 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 177 { 178 struct kse_execve_args args; 179 struct image_args iargs; 180 struct proc *p; 181 struct thread *td2; 182 struct kse_upcall *ku; 183 struct kse_thr_mailbox *tmbx; 184 uint32_t flags; 185 int error; 186 187 p = td->td_proc; 188 189 if (!(p->p_flag & P_SA)) 190 return (EINVAL); 191 192 switch (uap->cmd) { 193 case KSE_INTR_SENDSIG: 194 if (uap->data < 0 || uap->data > _SIG_MAXSIG) 195 return (EINVAL); 196 case KSE_INTR_INTERRUPT: 197 case KSE_INTR_RESTART: 198 PROC_LOCK(p); 199 mtx_lock_spin(&sched_lock); 200 FOREACH_THREAD_IN_PROC(p, td2) { 201 if (td2->td_mailbox == uap->tmbx) 202 break; 203 } 204 if (td2 == NULL) { 205 mtx_unlock_spin(&sched_lock); 206 PROC_UNLOCK(p); 207 return (ESRCH); 208 } 209 if (uap->cmd == KSE_INTR_SENDSIG) { 210 if (uap->data > 0) { 211 td2->td_flags &= ~TDF_INTERRUPT; 212 mtx_unlock_spin(&sched_lock); 213 tdsignal(td2, (int)uap->data, SIGTARGET_TD); 214 } else { 215 mtx_unlock_spin(&sched_lock); 216 } 217 } else { 218 td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; 219 if (TD_CAN_UNBIND(td2)) 220 td2->td_upcall->ku_flags |= KUF_DOUPCALL; 221 if (uap->cmd == KSE_INTR_INTERRUPT) 222 td2->td_intrval = EINTR; 223 else 224 td2->td_intrval = ERESTART; 225 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) 226 sleepq_abort(td2); 227 mtx_unlock_spin(&sched_lock); 228 } 229 PROC_UNLOCK(p); 230 break; 231 case KSE_INTR_SIGEXIT: 232 if (uap->data < 1 || uap->data > _SIG_MAXSIG) 233 return (EINVAL); 234 PROC_LOCK(p); 235 sigexit(td, (int)uap->data); 236 break; 237 238 case KSE_INTR_DBSUSPEND: 239 /* this sub-function is only for bound thread */ 240 if (td->td_pflags & TDP_SA) 241 return (EINVAL); 242 ku = td->td_upcall; 243 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 244 if (tmbx == NULL || tmbx == (void *)-1) 245 return (EINVAL); 246 flags = 0; 247 while ((p->p_flag & P_TRACED) && !(p->p_flag & P_SINGLE_EXIT)) { 248 flags = fuword32(&tmbx->tm_dflags); 249 if (!(flags & TMDF_SUSPEND)) 250 break; 251 PROC_LOCK(p); 252 mtx_lock_spin(&sched_lock); 253 thread_stopped(p); 254 thread_suspend_one(td); 255 PROC_UNLOCK(p); 256 mi_switch(SW_VOL, NULL); 257 mtx_unlock_spin(&sched_lock); 258 } 259 return (0); 260 261 case KSE_INTR_EXECVE: 262 error = copyin((void *)uap->data, &args, sizeof(args)); 263 if (error) 264 return (error); 265 error = exec_copyin_args(&iargs, args.path, UIO_USERSPACE, 266 args.argv, args.envp); 267 if (error == 0) 268 error = kern_execve(td, &iargs, NULL); 269 exec_free_args(&iargs); 270 if (error == 0) { 271 PROC_LOCK(p); 272 SIGSETOR(td->td_siglist, args.sigpend); 273 PROC_UNLOCK(p); 274 kern_sigprocmask(td, SIG_SETMASK, &args.sigmask, NULL, 275 0); 276 } 277 return (error); 278 279 default: 280 return (EINVAL); 281 } 282 return (0); 283 } 284 285 /* 286 struct kse_exit_args { 287 register_t dummy; 288 }; 289 */ 290 int 291 kse_exit(struct thread *td, struct kse_exit_args *uap) 292 { 293 struct proc *p; 294 struct ksegrp *kg; 295 struct kse_upcall *ku, *ku2; 296 int error, count; 297 298 p = td->td_proc; 299 /* 300 * Ensure that this is only called from the UTS 301 */ 302 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 303 return (EINVAL); 304 305 kg = td->td_ksegrp; 306 count = 0; 307 308 /* 309 * Calculate the existing non-exiting upcalls in this ksegroup. 310 * If we are the last upcall but there are still other threads, 311 * then do not exit. We need the other threads to be able to 312 * complete whatever they are doing. 313 * XXX This relies on the userland knowing what to do if we return. 314 * It may be a better choice to convert ourselves into a kse_release 315 * ( or similar) and wait in the kernel to be needed. 316 */ 317 PROC_LOCK(p); 318 mtx_lock_spin(&sched_lock); 319 FOREACH_UPCALL_IN_GROUP(kg, ku2) { 320 if (ku2->ku_flags & KUF_EXITING) 321 count++; 322 } 323 if ((kg->kg_numupcalls - count) == 1 && 324 (kg->kg_numthreads > 1)) { 325 mtx_unlock_spin(&sched_lock); 326 PROC_UNLOCK(p); 327 return (EDEADLK); 328 } 329 ku->ku_flags |= KUF_EXITING; 330 mtx_unlock_spin(&sched_lock); 331 PROC_UNLOCK(p); 332 333 /* 334 * Mark the UTS mailbox as having been finished with. 335 * If that fails then just go for a segfault. 336 * XXX need to check it that can be deliverred without a mailbox. 337 */ 338 error = suword32(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); 339 if (!(td->td_pflags & TDP_SA)) 340 if (suword32(&td->td_mailbox->tm_lwp, 0)) 341 error = EFAULT; 342 PROC_LOCK(p); 343 if (error) 344 psignal(p, SIGSEGV); 345 mtx_lock_spin(&sched_lock); 346 upcall_remove(td); 347 if (p->p_numthreads != 1) { 348 /* 349 * If we are not the last thread, but we are the last 350 * thread in this ksegrp, then by definition this is not 351 * the last group and we need to clean it up as well. 352 * thread_exit will clean up the kseg as needed. 353 */ 354 thread_stopped(p); 355 thread_exit(); 356 /* NOTREACHED */ 357 } 358 /* 359 * This is the last thread. Just return to the user. 360 * We know that there is only one ksegrp too, as any others 361 * would have been discarded in previous calls to thread_exit(). 362 * Effectively we have left threading mode.. 363 * The only real thing left to do is ensure that the 364 * scheduler sets out concurrency back to 1 as that may be a 365 * resource leak otherwise. 366 * This is an A[PB]I issue.. what SHOULD we do? 367 * One possibility is to return to the user. It may not cope well. 368 * The other possibility would be to let the process exit. 369 */ 370 thread_unthread(td); 371 mtx_unlock_spin(&sched_lock); 372 PROC_UNLOCK(p); 373 #if 1 374 return (0); 375 #else 376 exit1(td, 0); 377 #endif 378 } 379 380 /* 381 * Either becomes an upcall or waits for an awakening event and 382 * then becomes an upcall. Only error cases return. 383 */ 384 /* 385 struct kse_release_args { 386 struct timespec *timeout; 387 }; 388 */ 389 int 390 kse_release(struct thread *td, struct kse_release_args *uap) 391 { 392 struct proc *p; 393 struct ksegrp *kg; 394 struct kse_upcall *ku; 395 struct timespec timeout; 396 struct timeval tv; 397 sigset_t sigset; 398 int error; 399 400 p = td->td_proc; 401 kg = td->td_ksegrp; 402 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 403 return (EINVAL); 404 if (uap->timeout != NULL) { 405 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 406 return (error); 407 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 408 } 409 if (td->td_pflags & TDP_SA) 410 td->td_pflags |= TDP_UPCALLING; 411 else { 412 ku->ku_mflags = fuword32(&ku->ku_mailbox->km_flags); 413 if (ku->ku_mflags == -1) { 414 PROC_LOCK(p); 415 sigexit(td, SIGSEGV); 416 } 417 } 418 PROC_LOCK(p); 419 if (ku->ku_mflags & KMF_WAITSIGEVENT) { 420 /* UTS wants to wait for signal event */ 421 if (!(p->p_flag & P_SIGEVENT) && 422 !(ku->ku_flags & KUF_DOUPCALL)) { 423 td->td_kflags |= TDK_KSERELSIG; 424 error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, 425 "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); 426 td->td_kflags &= ~(TDK_KSERELSIG | TDK_WAKEUP); 427 } 428 p->p_flag &= ~P_SIGEVENT; 429 sigset = p->p_siglist; 430 PROC_UNLOCK(p); 431 error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, 432 sizeof(sigset)); 433 } else { 434 if ((ku->ku_flags & KUF_DOUPCALL) == 0 && 435 ((ku->ku_mflags & KMF_NOCOMPLETED) || 436 (kg->kg_completed == NULL))) { 437 kg->kg_upsleeps++; 438 td->td_kflags |= TDK_KSEREL; 439 error = msleep(&kg->kg_completed, &p->p_mtx, 440 PPAUSE|PCATCH, "kserel", 441 (uap->timeout ? tvtohz(&tv) : 0)); 442 td->td_kflags &= ~(TDK_KSEREL | TDK_WAKEUP); 443 kg->kg_upsleeps--; 444 } 445 PROC_UNLOCK(p); 446 } 447 if (ku->ku_flags & KUF_DOUPCALL) { 448 mtx_lock_spin(&sched_lock); 449 ku->ku_flags &= ~KUF_DOUPCALL; 450 mtx_unlock_spin(&sched_lock); 451 } 452 return (0); 453 } 454 455 /* struct kse_wakeup_args { 456 struct kse_mailbox *mbx; 457 }; */ 458 int 459 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 460 { 461 struct proc *p; 462 struct ksegrp *kg; 463 struct kse_upcall *ku; 464 struct thread *td2; 465 466 p = td->td_proc; 467 td2 = NULL; 468 ku = NULL; 469 /* KSE-enabled processes only, please. */ 470 if (!(p->p_flag & P_SA)) 471 return (EINVAL); 472 PROC_LOCK(p); 473 mtx_lock_spin(&sched_lock); 474 if (uap->mbx) { 475 FOREACH_KSEGRP_IN_PROC(p, kg) { 476 FOREACH_UPCALL_IN_GROUP(kg, ku) { 477 if (ku->ku_mailbox == uap->mbx) 478 break; 479 } 480 if (ku) 481 break; 482 } 483 } else { 484 kg = td->td_ksegrp; 485 if (kg->kg_upsleeps) { 486 mtx_unlock_spin(&sched_lock); 487 wakeup(&kg->kg_completed); 488 PROC_UNLOCK(p); 489 return (0); 490 } 491 ku = TAILQ_FIRST(&kg->kg_upcalls); 492 } 493 if (ku == NULL) { 494 mtx_unlock_spin(&sched_lock); 495 PROC_UNLOCK(p); 496 return (ESRCH); 497 } 498 if ((td2 = ku->ku_owner) == NULL) { 499 mtx_unlock_spin(&sched_lock); 500 panic("%s: no owner", __func__); 501 } else if (td2->td_kflags & (TDK_KSEREL | TDK_KSERELSIG)) { 502 mtx_unlock_spin(&sched_lock); 503 if (!(td2->td_kflags & TDK_WAKEUP)) { 504 td2->td_kflags |= TDK_WAKEUP; 505 if (td2->td_kflags & TDK_KSEREL) 506 sleepq_remove(td2, &kg->kg_completed); 507 else 508 sleepq_remove(td2, &p->p_siglist); 509 } 510 } else { 511 ku->ku_flags |= KUF_DOUPCALL; 512 mtx_unlock_spin(&sched_lock); 513 } 514 PROC_UNLOCK(p); 515 return (0); 516 } 517 518 /* 519 * No new KSEG: first call: use current KSE, don't schedule an upcall 520 * All other situations, do allocate max new KSEs and schedule an upcall. 521 * 522 * XXX should be changed so that 'first' behaviour lasts for as long 523 * as you have not made a kse in this ksegrp. i.e. as long as we do not have 524 * a mailbox.. 525 */ 526 /* struct kse_create_args { 527 struct kse_mailbox *mbx; 528 int newgroup; 529 }; */ 530 int 531 kse_create(struct thread *td, struct kse_create_args *uap) 532 { 533 struct ksegrp *newkg; 534 struct ksegrp *kg; 535 struct proc *p; 536 struct kse_mailbox mbx; 537 struct kse_upcall *newku; 538 int err, ncpus, sa = 0, first = 0; 539 struct thread *newtd; 540 541 p = td->td_proc; 542 kg = td->td_ksegrp; 543 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 544 return (err); 545 546 ncpus = mp_ncpus; 547 if (virtual_cpu != 0) 548 ncpus = virtual_cpu; 549 /* 550 * If the new UTS mailbox says that this 551 * will be a BOUND lwp, then it had better 552 * have its thread mailbox already there. 553 * In addition, this ksegrp will be limited to 554 * a concurrency of 1. There is more on this later. 555 */ 556 if (mbx.km_flags & KMF_BOUND) { 557 if (mbx.km_curthread == NULL) 558 return (EINVAL); 559 ncpus = 1; 560 } else { 561 sa = TDP_SA; 562 } 563 564 PROC_LOCK(p); 565 /* 566 * Processes using the other threading model can't 567 * suddenly start calling this one 568 */ 569 if ((p->p_flag & (P_SA|P_HADTHREADS)) == P_HADTHREADS) { 570 PROC_UNLOCK(p); 571 return (EINVAL); 572 } 573 574 /* 575 * Limit it to NCPU upcall contexts per ksegrp in any case. 576 * There is a small race here as we don't hold proclock 577 * until we inc the ksegrp count, but it's not really a big problem 578 * if we get one too many, but we save a proc lock. 579 */ 580 if ((!uap->newgroup) && (kg->kg_numupcalls >= ncpus)) { 581 PROC_UNLOCK(p); 582 return (EPROCLIM); 583 } 584 585 if (!(p->p_flag & P_SA)) { 586 first = 1; 587 p->p_flag |= P_SA|P_HADTHREADS; 588 } 589 590 PROC_UNLOCK(p); 591 /* 592 * Now pay attention! 593 * If we are going to be bound, then we need to be either 594 * a new group, or the first call ever. In either 595 * case we will be creating (or be) the only thread in a group. 596 * and the concurrency will be set to 1. 597 * This is not quite right, as we may still make ourself 598 * bound after making other ksegrps but it will do for now. 599 * The library will only try do this much. 600 */ 601 if (!sa && !(uap->newgroup || first)) 602 return (EINVAL); 603 604 if (uap->newgroup) { 605 newkg = ksegrp_alloc(); 606 bzero(&newkg->kg_startzero, 607 __rangeof(struct ksegrp, kg_startzero, kg_endzero)); 608 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 609 __rangeof(struct ksegrp, kg_startcopy, kg_endcopy)); 610 sched_init_concurrency(newkg); 611 PROC_LOCK(p); 612 if (p->p_numksegrps >= max_groups_per_proc) { 613 PROC_UNLOCK(p); 614 ksegrp_free(newkg); 615 return (EPROCLIM); 616 } 617 ksegrp_link(newkg, p); 618 mtx_lock_spin(&sched_lock); 619 sched_fork_ksegrp(td, newkg); 620 mtx_unlock_spin(&sched_lock); 621 PROC_UNLOCK(p); 622 } else { 623 /* 624 * We want to make a thread in our own ksegrp. 625 * If we are just the first call, either kind 626 * is ok, but if not then either we must be 627 * already an upcallable thread to make another, 628 * or a bound thread to make one of those. 629 * Once again, not quite right but good enough for now.. XXXKSE 630 */ 631 if (!first && ((td->td_pflags & TDP_SA) != sa)) 632 return (EINVAL); 633 634 newkg = kg; 635 } 636 637 /* 638 * This test is a bit "indirect". 639 * It might simplify things if we made a direct way of testing 640 * if a ksegrp has been worked on before. 641 * In the case of a bound request and the concurrency being set to 642 * one, the concurrency will already be 1 so it's just inefficient 643 * but not dangerous to call this again. XXX 644 */ 645 if (newkg->kg_numupcalls == 0) { 646 /* 647 * Initialize KSE group with the appropriate 648 * concurrency. 649 * 650 * For a multiplexed group, create as as much concurrency 651 * as the number of physical cpus. 652 * This increases concurrency in the kernel even if the 653 * userland is not MP safe and can only run on a single CPU. 654 * In an ideal world, every physical cpu should execute a 655 * thread. If there is enough concurrency, threads in the 656 * kernel can be executed parallel on different cpus at 657 * full speed without being restricted by the number of 658 * upcalls the userland provides. 659 * Adding more upcall structures only increases concurrency 660 * in userland. 661 * 662 * For a bound thread group, because there is only one thread 663 * in the group, we only set the concurrency for the group 664 * to 1. A thread in this kind of group will never schedule 665 * an upcall when blocked. This simulates pthread system 666 * scope thread behaviour. 667 */ 668 sched_set_concurrency(newkg, ncpus); 669 } 670 /* 671 * Even bound LWPs get a mailbox and an upcall to hold it. 672 */ 673 newku = upcall_alloc(); 674 newku->ku_mailbox = uap->mbx; 675 newku->ku_func = mbx.km_func; 676 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 677 678 /* 679 * For the first call this may not have been set. 680 * Of course nor may it actually be needed. 681 */ 682 if (td->td_standin == NULL) 683 thread_alloc_spare(td); 684 685 PROC_LOCK(p); 686 mtx_lock_spin(&sched_lock); 687 if (newkg->kg_numupcalls >= ncpus) { 688 mtx_unlock_spin(&sched_lock); 689 PROC_UNLOCK(p); 690 upcall_free(newku); 691 return (EPROCLIM); 692 } 693 694 /* 695 * If we are the first time, and a normal thread, 696 * then transfer all the signals back to the 'process'. 697 * SA threading will make a special thread to handle them. 698 */ 699 if (first && sa) { 700 SIGSETOR(p->p_siglist, td->td_siglist); 701 SIGEMPTYSET(td->td_siglist); 702 SIGFILLSET(td->td_sigmask); 703 SIG_CANTMASK(td->td_sigmask); 704 } 705 706 /* 707 * Make the new upcall available to the ksegrp. 708 * It may or may not use it, but it's available. 709 */ 710 upcall_link(newku, newkg); 711 PROC_UNLOCK(p); 712 if (mbx.km_quantum) 713 newkg->kg_upquantum = max(1, mbx.km_quantum / tick); 714 715 /* 716 * Each upcall structure has an owner thread, find which 717 * one owns it. 718 */ 719 if (uap->newgroup) { 720 /* 721 * Because the new ksegrp hasn't a thread, 722 * create an initial upcall thread to own it. 723 */ 724 newtd = thread_schedule_upcall(td, newku); 725 } else { 726 /* 727 * If the current thread hasn't an upcall structure, 728 * just assign the upcall to it. 729 * It'll just return. 730 */ 731 if (td->td_upcall == NULL) { 732 newku->ku_owner = td; 733 td->td_upcall = newku; 734 newtd = td; 735 } else { 736 /* 737 * Create a new upcall thread to own it. 738 */ 739 newtd = thread_schedule_upcall(td, newku); 740 } 741 } 742 mtx_unlock_spin(&sched_lock); 743 744 /* 745 * Let the UTS instance know its LWPID. 746 * It doesn't really care. But the debugger will. 747 */ 748 suword32(&newku->ku_mailbox->km_lwp, newtd->td_tid); 749 750 /* 751 * In the same manner, if the UTS has a current user thread, 752 * then it is also running on this LWP so set it as well. 753 * The library could do that of course.. but why not.. 754 */ 755 if (mbx.km_curthread) 756 suword32(&mbx.km_curthread->tm_lwp, newtd->td_tid); 757 758 759 if (sa) { 760 newtd->td_pflags |= TDP_SA; 761 } else { 762 newtd->td_pflags &= ~TDP_SA; 763 764 /* 765 * Since a library will use the mailbox pointer to 766 * identify even a bound thread, and the mailbox pointer 767 * will never be allowed to change after this syscall 768 * for a bound thread, set it here so the library can 769 * find the thread after the syscall returns. 770 */ 771 newtd->td_mailbox = mbx.km_curthread; 772 773 if (newtd != td) { 774 /* 775 * If we did create a new thread then 776 * make sure it goes to the right place 777 * when it starts up, and make sure that it runs 778 * at full speed when it gets there. 779 * thread_schedule_upcall() copies all cpu state 780 * to the new thread, so we should clear single step 781 * flag here. 782 */ 783 cpu_set_upcall_kse(newtd, newku->ku_func, 784 newku->ku_mailbox, &newku->ku_stack); 785 if (p->p_flag & P_TRACED) 786 ptrace_clear_single_step(newtd); 787 } 788 } 789 790 /* 791 * If we are starting a new thread, kick it off. 792 */ 793 if (newtd != td) { 794 mtx_lock_spin(&sched_lock); 795 setrunqueue(newtd, SRQ_BORING); 796 mtx_unlock_spin(&sched_lock); 797 } 798 return (0); 799 } 800 801 /* 802 * Initialize global thread allocation resources. 803 */ 804 void 805 kseinit(void) 806 { 807 808 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 809 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 810 } 811 812 /* 813 * Stash an embarasingly extra upcall into the zombie upcall queue. 814 */ 815 816 void 817 upcall_stash(struct kse_upcall *ku) 818 { 819 mtx_lock_spin(&kse_zombie_lock); 820 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 821 mtx_unlock_spin(&kse_zombie_lock); 822 } 823 824 /* 825 * Reap zombie kse resource. 826 */ 827 void 828 kse_GC(void) 829 { 830 struct kse_upcall *ku_first, *ku_next; 831 832 /* 833 * Don't even bother to lock if none at this instant, 834 * we really don't care about the next instant.. 835 */ 836 if (!TAILQ_EMPTY(&zombie_upcalls)) { 837 mtx_lock_spin(&kse_zombie_lock); 838 ku_first = TAILQ_FIRST(&zombie_upcalls); 839 if (ku_first) 840 TAILQ_INIT(&zombie_upcalls); 841 mtx_unlock_spin(&kse_zombie_lock); 842 while (ku_first) { 843 ku_next = TAILQ_NEXT(ku_first, ku_link); 844 upcall_free(ku_first); 845 ku_first = ku_next; 846 } 847 } 848 } 849 850 /* 851 * Store the thread context in the UTS's mailbox. 852 * then add the mailbox at the head of a list we are building in user space. 853 * The list is anchored in the ksegrp structure. 854 */ 855 int 856 thread_export_context(struct thread *td, int willexit) 857 { 858 struct proc *p; 859 struct ksegrp *kg; 860 uintptr_t mbx; 861 void *addr; 862 int error = 0, sig; 863 mcontext_t mc; 864 865 p = td->td_proc; 866 kg = td->td_ksegrp; 867 868 /* 869 * Post sync signal, or process SIGKILL and SIGSTOP. 870 * For sync signal, it is only possible when the signal is not 871 * caught by userland or process is being debugged. 872 */ 873 PROC_LOCK(p); 874 if (td->td_flags & TDF_NEEDSIGCHK) { 875 mtx_lock_spin(&sched_lock); 876 td->td_flags &= ~TDF_NEEDSIGCHK; 877 mtx_unlock_spin(&sched_lock); 878 mtx_lock(&p->p_sigacts->ps_mtx); 879 while ((sig = cursig(td)) != 0) 880 postsig(sig); 881 mtx_unlock(&p->p_sigacts->ps_mtx); 882 } 883 if (willexit) 884 SIGFILLSET(td->td_sigmask); 885 PROC_UNLOCK(p); 886 887 /* Export the user/machine context. */ 888 get_mcontext(td, &mc, 0); 889 addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); 890 error = copyout(&mc, addr, sizeof(mcontext_t)); 891 if (error) 892 goto bad; 893 894 addr = (caddr_t)(&td->td_mailbox->tm_lwp); 895 if (suword32(addr, 0)) { 896 error = EFAULT; 897 goto bad; 898 } 899 900 /* Get address in latest mbox of list pointer */ 901 addr = (void *)(&td->td_mailbox->tm_next); 902 /* 903 * Put the saved address of the previous first 904 * entry into this one 905 */ 906 for (;;) { 907 mbx = (uintptr_t)kg->kg_completed; 908 if (suword(addr, mbx)) { 909 error = EFAULT; 910 goto bad; 911 } 912 PROC_LOCK(p); 913 if (mbx == (uintptr_t)kg->kg_completed) { 914 kg->kg_completed = td->td_mailbox; 915 /* 916 * The thread context may be taken away by 917 * other upcall threads when we unlock 918 * process lock. it's no longer valid to 919 * use it again in any other places. 920 */ 921 td->td_mailbox = NULL; 922 PROC_UNLOCK(p); 923 break; 924 } 925 PROC_UNLOCK(p); 926 } 927 td->td_usticks = 0; 928 return (0); 929 930 bad: 931 PROC_LOCK(p); 932 sigexit(td, SIGILL); 933 return (error); 934 } 935 936 /* 937 * Take the list of completed mailboxes for this KSEGRP and put them on this 938 * upcall's mailbox as it's the next one going up. 939 */ 940 static int 941 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 942 { 943 struct proc *p = kg->kg_proc; 944 void *addr; 945 uintptr_t mbx; 946 947 addr = (void *)(&ku->ku_mailbox->km_completed); 948 for (;;) { 949 mbx = (uintptr_t)kg->kg_completed; 950 if (suword(addr, mbx)) { 951 PROC_LOCK(p); 952 psignal(p, SIGSEGV); 953 PROC_UNLOCK(p); 954 return (EFAULT); 955 } 956 PROC_LOCK(p); 957 if (mbx == (uintptr_t)kg->kg_completed) { 958 kg->kg_completed = NULL; 959 PROC_UNLOCK(p); 960 break; 961 } 962 PROC_UNLOCK(p); 963 } 964 return (0); 965 } 966 967 /* 968 * This function should be called at statclock interrupt time 969 */ 970 int 971 thread_statclock(int user) 972 { 973 struct thread *td = curthread; 974 975 if (!(td->td_pflags & TDP_SA)) 976 return (0); 977 if (user) { 978 /* Current always do via ast() */ 979 mtx_lock_spin(&sched_lock); 980 td->td_flags |= TDF_ASTPENDING; 981 mtx_unlock_spin(&sched_lock); 982 td->td_uuticks++; 983 } else if (td->td_mailbox != NULL) 984 td->td_usticks++; 985 return (0); 986 } 987 988 /* 989 * Export state clock ticks for userland 990 */ 991 static int 992 thread_update_usr_ticks(struct thread *td) 993 { 994 struct proc *p = td->td_proc; 995 caddr_t addr; 996 u_int uticks; 997 998 if (td->td_mailbox == NULL) 999 return (-1); 1000 1001 if ((uticks = td->td_uuticks) != 0) { 1002 td->td_uuticks = 0; 1003 addr = (caddr_t)&td->td_mailbox->tm_uticks; 1004 if (suword32(addr, uticks+fuword32(addr))) 1005 goto error; 1006 } 1007 if ((uticks = td->td_usticks) != 0) { 1008 td->td_usticks = 0; 1009 addr = (caddr_t)&td->td_mailbox->tm_sticks; 1010 if (suword32(addr, uticks+fuword32(addr))) 1011 goto error; 1012 } 1013 return (0); 1014 1015 error: 1016 PROC_LOCK(p); 1017 psignal(p, SIGSEGV); 1018 PROC_UNLOCK(p); 1019 return (-2); 1020 } 1021 1022 /* 1023 * This function is intended to be used to initialize a spare thread 1024 * for upcall. Initialize thread's large data area outside sched_lock 1025 * for thread_schedule_upcall(). The crhold is also here to get it out 1026 * from the schedlock as it has a mutex op itself. 1027 * XXX BUG.. we need to get the cr ref after the thread has 1028 * checked and chenged its own, not 6 months before... 1029 */ 1030 void 1031 thread_alloc_spare(struct thread *td) 1032 { 1033 struct thread *spare; 1034 1035 if (td->td_standin) 1036 return; 1037 spare = thread_alloc(); 1038 td->td_standin = spare; 1039 bzero(&spare->td_startzero, 1040 __rangeof(struct thread, td_startzero, td_endzero)); 1041 spare->td_proc = td->td_proc; 1042 spare->td_ucred = crhold(td->td_ucred); 1043 } 1044 1045 /* 1046 * Create a thread and schedule it for upcall on the KSE given. 1047 * Use our thread's standin so that we don't have to allocate one. 1048 */ 1049 struct thread * 1050 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1051 { 1052 struct thread *td2; 1053 1054 mtx_assert(&sched_lock, MA_OWNED); 1055 1056 /* 1057 * Schedule an upcall thread on specified kse_upcall, 1058 * the kse_upcall must be free. 1059 * td must have a spare thread. 1060 */ 1061 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1062 if ((td2 = td->td_standin) != NULL) { 1063 td->td_standin = NULL; 1064 } else { 1065 panic("no reserve thread when scheduling an upcall"); 1066 return (NULL); 1067 } 1068 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1069 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1070 /* 1071 * Bzero already done in thread_alloc_spare() because we can't 1072 * do the crhold here because we are in schedlock already. 1073 */ 1074 bcopy(&td->td_startcopy, &td2->td_startcopy, 1075 __rangeof(struct thread, td_startcopy, td_endcopy)); 1076 thread_link(td2, ku->ku_ksegrp); 1077 /* inherit parts of blocked thread's context as a good template */ 1078 cpu_set_upcall(td2, td); 1079 /* Let the new thread become owner of the upcall */ 1080 ku->ku_owner = td2; 1081 td2->td_upcall = ku; 1082 td2->td_flags = 0; 1083 td2->td_pflags = TDP_SA|TDP_UPCALLING; 1084 td2->td_state = TDS_CAN_RUN; 1085 td2->td_inhibitors = 0; 1086 SIGFILLSET(td2->td_sigmask); 1087 SIG_CANTMASK(td2->td_sigmask); 1088 sched_fork_thread(td, td2); 1089 return (td2); /* bogus.. should be a void function */ 1090 } 1091 1092 /* 1093 * It is only used when thread generated a trap and process is being 1094 * debugged. 1095 */ 1096 void 1097 thread_signal_add(struct thread *td, int sig) 1098 { 1099 struct proc *p; 1100 siginfo_t siginfo; 1101 struct sigacts *ps; 1102 int error; 1103 1104 p = td->td_proc; 1105 PROC_LOCK_ASSERT(p, MA_OWNED); 1106 ps = p->p_sigacts; 1107 mtx_assert(&ps->ps_mtx, MA_OWNED); 1108 1109 cpu_thread_siginfo(sig, 0, &siginfo); 1110 mtx_unlock(&ps->ps_mtx); 1111 SIGADDSET(td->td_sigmask, sig); 1112 PROC_UNLOCK(p); 1113 error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); 1114 if (error) { 1115 PROC_LOCK(p); 1116 sigexit(td, SIGSEGV); 1117 } 1118 PROC_LOCK(p); 1119 mtx_lock(&ps->ps_mtx); 1120 } 1121 #include "opt_sched.h" 1122 struct thread * 1123 thread_switchout(struct thread *td, int flags, struct thread *nextthread) 1124 { 1125 struct kse_upcall *ku; 1126 struct thread *td2; 1127 1128 mtx_assert(&sched_lock, MA_OWNED); 1129 1130 /* 1131 * If the outgoing thread is in threaded group and has never 1132 * scheduled an upcall, decide whether this is a short 1133 * or long term event and thus whether or not to schedule 1134 * an upcall. 1135 * If it is a short term event, just suspend it in 1136 * a way that takes its KSE with it. 1137 * Select the events for which we want to schedule upcalls. 1138 * For now it's just sleep or if thread is suspended but 1139 * process wide suspending flag is not set (debugger 1140 * suspends thread). 1141 * XXXKSE eventually almost any inhibition could do. 1142 */ 1143 if (TD_CAN_UNBIND(td) && (td->td_standin) && 1144 (TD_ON_SLEEPQ(td) || (TD_IS_SUSPENDED(td) && 1145 !P_SHOULDSTOP(td->td_proc)))) { 1146 /* 1147 * Release ownership of upcall, and schedule an upcall 1148 * thread, this new upcall thread becomes the owner of 1149 * the upcall structure. It will be ahead of us in the 1150 * run queue, so as we are stopping, it should either 1151 * start up immediatly, or at least before us if 1152 * we release our slot. 1153 */ 1154 ku = td->td_upcall; 1155 ku->ku_owner = NULL; 1156 td->td_upcall = NULL; 1157 td->td_pflags &= ~TDP_CAN_UNBIND; 1158 td2 = thread_schedule_upcall(td, ku); 1159 if (flags & SW_INVOL || nextthread) { 1160 setrunqueue(td2, SRQ_YIELDING); 1161 } else { 1162 /* Keep up with reality.. we have one extra thread 1163 * in the picture.. and it's 'running'. 1164 */ 1165 return td2; 1166 } 1167 } 1168 return (nextthread); 1169 } 1170 1171 /* 1172 * Setup done on the thread when it enters the kernel. 1173 */ 1174 void 1175 thread_user_enter(struct thread *td) 1176 { 1177 struct proc *p = td->td_proc; 1178 struct ksegrp *kg; 1179 struct kse_upcall *ku; 1180 struct kse_thr_mailbox *tmbx; 1181 uint32_t flags; 1182 1183 /* 1184 * First check that we shouldn't just abort. we 1185 * can suspend it here or just exit. 1186 */ 1187 if (__predict_false(P_SHOULDSTOP(p))) { 1188 PROC_LOCK(p); 1189 thread_suspend_check(0); 1190 PROC_UNLOCK(p); 1191 } 1192 1193 if (!(td->td_pflags & TDP_SA)) 1194 return; 1195 1196 /* 1197 * If we are doing a syscall in a KSE environment, 1198 * note where our mailbox is. 1199 */ 1200 1201 kg = td->td_ksegrp; 1202 ku = td->td_upcall; 1203 1204 KASSERT(ku != NULL, ("no upcall owned")); 1205 KASSERT(ku->ku_owner == td, ("wrong owner")); 1206 KASSERT(!TD_CAN_UNBIND(td), ("can unbind")); 1207 1208 if (td->td_standin == NULL) 1209 thread_alloc_spare(td); 1210 ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); 1211 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1212 if ((tmbx == NULL) || (tmbx == (void *)-1L) || 1213 (ku->ku_mflags & KMF_NOUPCALL)) { 1214 td->td_mailbox = NULL; 1215 } else { 1216 flags = fuword32(&tmbx->tm_flags); 1217 /* 1218 * On some architectures, TP register points to thread 1219 * mailbox but not points to kse mailbox, and userland 1220 * can not atomically clear km_curthread, but can 1221 * use TP register, and set TMF_NOUPCALL in thread 1222 * flag to indicate a critical region. 1223 */ 1224 if (flags & TMF_NOUPCALL) { 1225 td->td_mailbox = NULL; 1226 } else { 1227 td->td_mailbox = tmbx; 1228 td->td_pflags |= TDP_CAN_UNBIND; 1229 if (__predict_false(p->p_flag & P_TRACED)) { 1230 flags = fuword32(&tmbx->tm_dflags); 1231 if (flags & TMDF_SUSPEND) { 1232 mtx_lock_spin(&sched_lock); 1233 /* fuword can block, check again */ 1234 if (td->td_upcall) 1235 ku->ku_flags |= KUF_DOUPCALL; 1236 mtx_unlock_spin(&sched_lock); 1237 } 1238 } 1239 } 1240 } 1241 } 1242 1243 /* 1244 * The extra work we go through if we are a threaded process when we 1245 * return to userland. 1246 * 1247 * If we are a KSE process and returning to user mode, check for 1248 * extra work to do before we return (e.g. for more syscalls 1249 * to complete first). If we were in a critical section, we should 1250 * just return to let it finish. Same if we were in the UTS (in 1251 * which case the mailbox's context's busy indicator will be set). 1252 * The only traps we suport will have set the mailbox. 1253 * We will clear it here. 1254 */ 1255 int 1256 thread_userret(struct thread *td, struct trapframe *frame) 1257 { 1258 struct kse_upcall *ku; 1259 struct ksegrp *kg, *kg2; 1260 struct proc *p; 1261 struct timespec ts; 1262 int error = 0, upcalls, uts_crit; 1263 1264 /* Nothing to do with bound thread */ 1265 if (!(td->td_pflags & TDP_SA)) 1266 return (0); 1267 1268 /* 1269 * Update stat clock count for userland 1270 */ 1271 if (td->td_mailbox != NULL) { 1272 thread_update_usr_ticks(td); 1273 uts_crit = 0; 1274 } else { 1275 uts_crit = 1; 1276 } 1277 1278 p = td->td_proc; 1279 kg = td->td_ksegrp; 1280 ku = td->td_upcall; 1281 1282 /* 1283 * Optimisation: 1284 * This thread has not started any upcall. 1285 * If there is no work to report other than ourself, 1286 * then it can return direct to userland. 1287 */ 1288 if (TD_CAN_UNBIND(td)) { 1289 td->td_pflags &= ~TDP_CAN_UNBIND; 1290 if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1291 (kg->kg_completed == NULL) && 1292 (ku->ku_flags & KUF_DOUPCALL) == 0 && 1293 (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1294 nanotime(&ts); 1295 error = copyout(&ts, 1296 (caddr_t)&ku->ku_mailbox->km_timeofday, 1297 sizeof(ts)); 1298 td->td_mailbox = 0; 1299 ku->ku_mflags = 0; 1300 if (error) 1301 goto out; 1302 return (0); 1303 } 1304 thread_export_context(td, 0); 1305 /* 1306 * There is something to report, and we own an upcall 1307 * structure, we can go to userland. 1308 * Turn ourself into an upcall thread. 1309 */ 1310 td->td_pflags |= TDP_UPCALLING; 1311 } else if (td->td_mailbox && (ku == NULL)) { 1312 thread_export_context(td, 1); 1313 PROC_LOCK(p); 1314 if (kg->kg_upsleeps) 1315 wakeup(&kg->kg_completed); 1316 WITNESS_WARN(WARN_PANIC, &p->p_mtx.mtx_object, 1317 "thread exiting in userret"); 1318 mtx_lock_spin(&sched_lock); 1319 thread_stopped(p); 1320 thread_exit(); 1321 /* NOTREACHED */ 1322 } 1323 1324 KASSERT(ku != NULL, ("upcall is NULL")); 1325 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1326 1327 if (p->p_numthreads > max_threads_per_proc) { 1328 max_threads_hits++; 1329 PROC_LOCK(p); 1330 mtx_lock_spin(&sched_lock); 1331 p->p_maxthrwaits++; 1332 while (p->p_numthreads > max_threads_per_proc) { 1333 upcalls = 0; 1334 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1335 if (kg2->kg_numupcalls == 0) 1336 upcalls++; 1337 else 1338 upcalls += kg2->kg_numupcalls; 1339 } 1340 if (upcalls >= max_threads_per_proc) 1341 break; 1342 mtx_unlock_spin(&sched_lock); 1343 if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1344 "maxthreads", hz/10) != EWOULDBLOCK) { 1345 mtx_lock_spin(&sched_lock); 1346 break; 1347 } else { 1348 mtx_lock_spin(&sched_lock); 1349 } 1350 } 1351 p->p_maxthrwaits--; 1352 mtx_unlock_spin(&sched_lock); 1353 PROC_UNLOCK(p); 1354 } 1355 1356 if (td->td_pflags & TDP_UPCALLING) { 1357 uts_crit = 0; 1358 kg->kg_nextupcall = ticks + kg->kg_upquantum; 1359 /* 1360 * There is no more work to do and we are going to ride 1361 * this thread up to userland as an upcall. 1362 * Do the last parts of the setup needed for the upcall. 1363 */ 1364 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1365 td, td->td_proc->p_pid, td->td_proc->p_comm); 1366 1367 td->td_pflags &= ~TDP_UPCALLING; 1368 if (ku->ku_flags & KUF_DOUPCALL) { 1369 mtx_lock_spin(&sched_lock); 1370 ku->ku_flags &= ~KUF_DOUPCALL; 1371 mtx_unlock_spin(&sched_lock); 1372 } 1373 /* 1374 * Set user context to the UTS 1375 */ 1376 if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1377 cpu_set_upcall_kse(td, ku->ku_func, ku->ku_mailbox, 1378 &ku->ku_stack); 1379 if (p->p_flag & P_TRACED) 1380 ptrace_clear_single_step(td); 1381 error = suword32(&ku->ku_mailbox->km_lwp, 1382 td->td_tid); 1383 if (error) 1384 goto out; 1385 error = suword(&ku->ku_mailbox->km_curthread, 0); 1386 if (error) 1387 goto out; 1388 } 1389 1390 /* 1391 * Unhook the list of completed threads. 1392 * anything that completes after this gets to 1393 * come in next time. 1394 * Put the list of completed thread mailboxes on 1395 * this KSE's mailbox. 1396 */ 1397 if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1398 (error = thread_link_mboxes(kg, ku)) != 0) 1399 goto out; 1400 } 1401 if (!uts_crit) { 1402 nanotime(&ts); 1403 error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1404 } 1405 1406 out: 1407 if (error) { 1408 /* 1409 * Things are going to be so screwed we should just kill 1410 * the process. 1411 * how do we do that? 1412 */ 1413 PROC_LOCK(p); 1414 psignal(p, SIGSEGV); 1415 PROC_UNLOCK(p); 1416 } else { 1417 /* 1418 * Optimisation: 1419 * Ensure that we have a spare thread available, 1420 * for when we re-enter the kernel. 1421 */ 1422 if (td->td_standin == NULL) 1423 thread_alloc_spare(td); 1424 } 1425 1426 ku->ku_mflags = 0; 1427 td->td_mailbox = NULL; 1428 td->td_usticks = 0; 1429 return (error); /* go sync */ 1430 } 1431 1432 /* 1433 * called after ptrace resumed a process, force all 1434 * virtual CPUs to schedule upcall for SA process, 1435 * because debugger may have changed something in userland, 1436 * we should notice UTS as soon as possible. 1437 */ 1438 void 1439 thread_continued(struct proc *p) 1440 { 1441 struct ksegrp *kg; 1442 struct kse_upcall *ku; 1443 struct thread *td; 1444 1445 PROC_LOCK_ASSERT(p, MA_OWNED); 1446 KASSERT(P_SHOULDSTOP(p), ("process not stopped")); 1447 1448 if (!(p->p_flag & P_SA)) 1449 return; 1450 1451 if (p->p_flag & P_TRACED) { 1452 FOREACH_KSEGRP_IN_PROC(p, kg) { 1453 td = TAILQ_FIRST(&kg->kg_threads); 1454 if (td == NULL) 1455 continue; 1456 /* not a SA group, nothing to do */ 1457 if (!(td->td_pflags & TDP_SA)) 1458 continue; 1459 FOREACH_UPCALL_IN_GROUP(kg, ku) { 1460 mtx_lock_spin(&sched_lock); 1461 ku->ku_flags |= KUF_DOUPCALL; 1462 mtx_unlock_spin(&sched_lock); 1463 wakeup(&kg->kg_completed); 1464 } 1465 } 1466 } 1467 }
Cache object: 4aa3b68f87417ded174f0d84e3c6dcf2
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]