Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_condvar.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: releng/5.1/sys/kern/kern_condvar.c 114983 2003-05-13 20:36:02Z jhb $ 27 */ 28 29 #include "opt_ktrace.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/lock.h> 34 #include <sys/mutex.h> 35 #include <sys/proc.h> 36 #include <sys/kernel.h> 37 #include <sys/ktr.h> 38 #include <sys/condvar.h> 39 #include <sys/sched.h> 40 #include <sys/signalvar.h> 41 #include <sys/resourcevar.h> 42 #ifdef KTRACE 43 #include <sys/uio.h> 44 #include <sys/ktrace.h> 45 #endif 46 47 /* 48 * Common sanity checks for cv_wait* functions. 49 */ 50 #define CV_ASSERT(cvp, mp, td) do { \ 51 KASSERT((td) != NULL, ("%s: curthread NULL", __func__)); \ 52 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 53 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 54 KASSERT((mp) != NULL, ("%s: mp NULL", __func__)); \ 55 mtx_assert((mp), MA_OWNED | MA_NOTRECURSED); \ 56 } while (0) 57 58 #ifdef INVARIANTS 59 #define CV_WAIT_VALIDATE(cvp, mp) do { \ 60 if (TAILQ_EMPTY(&(cvp)->cv_waitq)) { \ 61 /* Only waiter. */ \ 62 (cvp)->cv_mtx = (mp); \ 63 } else { \ 64 /* \ 65 * Other waiter; assert that we're using the \ 66 * same mutex. \ 67 */ \ 68 KASSERT((cvp)->cv_mtx == (mp), \ 69 ("%s: Multiple mutexes", __func__)); \ 70 } \ 71 } while (0) 72 73 #define CV_SIGNAL_VALIDATE(cvp) do { \ 74 if (!TAILQ_EMPTY(&(cvp)->cv_waitq)) { \ 75 KASSERT(mtx_owned((cvp)->cv_mtx), \ 76 ("%s: Mutex not owned", __func__)); \ 77 } \ 78 } while (0) 79 80 #else 81 #define CV_WAIT_VALIDATE(cvp, mp) 82 #define CV_SIGNAL_VALIDATE(cvp) 83 #endif 84 85 static void cv_timedwait_end(void *arg); 86 87 /* 88 * Initialize a condition variable. Must be called before use. 89 */ 90 void 91 cv_init(struct cv *cvp, const char *desc) 92 { 93 94 TAILQ_INIT(&cvp->cv_waitq); 95 cvp->cv_mtx = NULL; 96 cvp->cv_description = desc; 97 } 98 99 /* 100 * Destroy a condition variable. The condition variable must be re-initialized 101 * in order to be re-used. 102 */ 103 void 104 cv_destroy(struct cv *cvp) 105 { 106 107 KASSERT(cv_waitq_empty(cvp), ("%s: cv_waitq non-empty", __func__)); 108 } 109 110 /* 111 * Common code for cv_wait* functions. All require sched_lock. 112 */ 113 114 /* 115 * Switch context. 116 */ 117 static __inline void 118 cv_switch(struct thread *td) 119 { 120 TD_SET_SLEEPING(td); 121 td->td_proc->p_stats->p_ru.ru_nvcsw++; 122 mi_switch(); 123 CTR3(KTR_PROC, "cv_switch: resume thread %p (pid %d, %s)", td, 124 td->td_proc->p_pid, td->td_proc->p_comm); 125 } 126 127 /* 128 * Switch context, catching signals. 129 */ 130 static __inline int 131 cv_switch_catch(struct thread *td) 132 { 133 struct proc *p; 134 int sig; 135 136 /* 137 * We put ourselves on the sleep queue and start our timeout before 138 * calling cursig, as we could stop there, and a wakeup or a SIGCONT (or 139 * both) could occur while we were stopped. A SIGCONT would cause us to 140 * be marked as TDS_SLP without resuming us, thus we must be ready for 141 * sleep when cursig is called. If the wakeup happens while we're 142 * stopped, td->td_wchan will be 0 upon return from cursig, 143 * and TD_ON_SLEEPQ() will return false. 144 */ 145 td->td_flags |= TDF_SINTR; 146 mtx_unlock_spin(&sched_lock); 147 p = td->td_proc; 148 PROC_LOCK(p); 149 mtx_lock(&p->p_sigacts->ps_mtx); 150 sig = cursig(td); 151 mtx_unlock(&p->p_sigacts->ps_mtx); 152 if (thread_suspend_check(1)) 153 sig = SIGSTOP; 154 mtx_lock_spin(&sched_lock); 155 PROC_UNLOCK(p); 156 if (sig != 0) { 157 if (TD_ON_SLEEPQ(td)) 158 cv_waitq_remove(td); 159 TD_SET_RUNNING(td); 160 } else if (TD_ON_SLEEPQ(td)) { 161 cv_switch(td); 162 } 163 td->td_flags &= ~TDF_SINTR; 164 165 return sig; 166 } 167 168 /* 169 * Add a thread to the wait queue of a condition variable. 170 */ 171 static __inline void 172 cv_waitq_add(struct cv *cvp, struct thread *td) 173 { 174 175 td->td_flags |= TDF_CVWAITQ; 176 TD_SET_ON_SLEEPQ(td); 177 td->td_wchan = cvp; 178 td->td_wmesg = cvp->cv_description; 179 CTR3(KTR_PROC, "cv_waitq_add: thread %p (pid %d, %s)", td, 180 td->td_proc->p_pid, td->td_proc->p_comm); 181 TAILQ_INSERT_TAIL(&cvp->cv_waitq, td, td_slpq); 182 sched_sleep(td, td->td_priority); 183 } 184 185 /* 186 * Wait on a condition variable. The current thread is placed on the condition 187 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 188 * condition variable will resume the thread. The mutex is released before 189 * sleeping and will be held on return. It is recommended that the mutex be 190 * held when cv_signal or cv_broadcast are called. 191 */ 192 void 193 cv_wait(struct cv *cvp, struct mtx *mp) 194 { 195 struct thread *td; 196 WITNESS_SAVE_DECL(mp); 197 198 td = curthread; 199 #ifdef KTRACE 200 if (KTRPOINT(td, KTR_CSW)) 201 ktrcsw(1, 0); 202 #endif 203 CV_ASSERT(cvp, mp, td); 204 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object, 205 "Waiting on \"%s\"", cvp->cv_description); 206 WITNESS_SAVE(&mp->mtx_object, mp); 207 208 if (cold ) { 209 /* 210 * During autoconfiguration, just give interrupts 211 * a chance, then just return. Don't run any other 212 * thread or panic below, in case this is the idle 213 * process and already asleep. 214 */ 215 return; 216 } 217 218 mtx_lock_spin(&sched_lock); 219 220 CV_WAIT_VALIDATE(cvp, mp); 221 222 DROP_GIANT(); 223 mtx_unlock(mp); 224 225 cv_waitq_add(cvp, td); 226 cv_switch(td); 227 228 mtx_unlock_spin(&sched_lock); 229 #ifdef KTRACE 230 if (KTRPOINT(td, KTR_CSW)) 231 ktrcsw(0, 0); 232 #endif 233 PICKUP_GIANT(); 234 mtx_lock(mp); 235 WITNESS_RESTORE(&mp->mtx_object, mp); 236 } 237 238 /* 239 * Wait on a condition variable, allowing interruption by signals. Return 0 if 240 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 241 * a signal was caught. If ERESTART is returned the system call should be 242 * restarted if possible. 243 */ 244 int 245 cv_wait_sig(struct cv *cvp, struct mtx *mp) 246 { 247 struct thread *td; 248 struct proc *p; 249 int rval; 250 int sig; 251 WITNESS_SAVE_DECL(mp); 252 253 td = curthread; 254 p = td->td_proc; 255 rval = 0; 256 #ifdef KTRACE 257 if (KTRPOINT(td, KTR_CSW)) 258 ktrcsw(1, 0); 259 #endif 260 CV_ASSERT(cvp, mp, td); 261 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object, 262 "Waiting on \"%s\"", cvp->cv_description); 263 WITNESS_SAVE(&mp->mtx_object, mp); 264 265 if (cold || panicstr) { 266 /* 267 * After a panic, or during autoconfiguration, just give 268 * interrupts a chance, then just return; don't run any other 269 * procs or panic below, in case this is the idle process and 270 * already asleep. 271 */ 272 return 0; 273 } 274 275 mtx_lock_spin(&sched_lock); 276 277 CV_WAIT_VALIDATE(cvp, mp); 278 279 DROP_GIANT(); 280 mtx_unlock(mp); 281 282 cv_waitq_add(cvp, td); 283 sig = cv_switch_catch(td); 284 285 mtx_unlock_spin(&sched_lock); 286 287 PROC_LOCK(p); 288 mtx_lock(&p->p_sigacts->ps_mtx); 289 if (sig == 0) 290 sig = cursig(td); /* XXXKSE */ 291 if (sig != 0) { 292 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig)) 293 rval = EINTR; 294 else 295 rval = ERESTART; 296 } 297 mtx_unlock(&p->p_sigacts->ps_mtx); 298 if (p->p_flag & P_WEXIT) 299 rval = EINTR; 300 PROC_UNLOCK(p); 301 302 #ifdef KTRACE 303 if (KTRPOINT(td, KTR_CSW)) 304 ktrcsw(0, 0); 305 #endif 306 PICKUP_GIANT(); 307 mtx_lock(mp); 308 WITNESS_RESTORE(&mp->mtx_object, mp); 309 310 return (rval); 311 } 312 313 /* 314 * Wait on a condition variable for at most timo/hz seconds. Returns 0 if the 315 * process was resumed by cv_signal or cv_broadcast, EWOULDBLOCK if the timeout 316 * expires. 317 */ 318 int 319 cv_timedwait(struct cv *cvp, struct mtx *mp, int timo) 320 { 321 struct thread *td; 322 int rval; 323 WITNESS_SAVE_DECL(mp); 324 325 td = curthread; 326 rval = 0; 327 #ifdef KTRACE 328 if (KTRPOINT(td, KTR_CSW)) 329 ktrcsw(1, 0); 330 #endif 331 CV_ASSERT(cvp, mp, td); 332 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object, 333 "Waiting on \"%s\"", cvp->cv_description); 334 WITNESS_SAVE(&mp->mtx_object, mp); 335 336 if (cold || panicstr) { 337 /* 338 * After a panic, or during autoconfiguration, just give 339 * interrupts a chance, then just return; don't run any other 340 * thread or panic below, in case this is the idle process and 341 * already asleep. 342 */ 343 return 0; 344 } 345 346 mtx_lock_spin(&sched_lock); 347 348 CV_WAIT_VALIDATE(cvp, mp); 349 350 DROP_GIANT(); 351 mtx_unlock(mp); 352 353 cv_waitq_add(cvp, td); 354 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td); 355 cv_switch(td); 356 357 if (td->td_flags & TDF_TIMEOUT) { 358 td->td_flags &= ~TDF_TIMEOUT; 359 rval = EWOULDBLOCK; 360 } else if (td->td_flags & TDF_TIMOFAIL) 361 td->td_flags &= ~TDF_TIMOFAIL; 362 else if (callout_stop(&td->td_slpcallout) == 0) { 363 /* 364 * Work around race with cv_timedwait_end similar to that 365 * between msleep and endtsleep. 366 * Go back to sleep. 367 */ 368 TD_SET_SLEEPING(td); 369 td->td_proc->p_stats->p_ru.ru_nivcsw++; 370 mi_switch(); 371 td->td_flags &= ~TDF_TIMOFAIL; 372 } 373 374 mtx_unlock_spin(&sched_lock); 375 #ifdef KTRACE 376 if (KTRPOINT(td, KTR_CSW)) 377 ktrcsw(0, 0); 378 #endif 379 PICKUP_GIANT(); 380 mtx_lock(mp); 381 WITNESS_RESTORE(&mp->mtx_object, mp); 382 383 return (rval); 384 } 385 386 /* 387 * Wait on a condition variable for at most timo/hz seconds, allowing 388 * interruption by signals. Returns 0 if the thread was resumed by cv_signal 389 * or cv_broadcast, EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if 390 * a signal was caught. 391 */ 392 int 393 cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo) 394 { 395 struct thread *td; 396 struct proc *p; 397 int rval; 398 int sig; 399 WITNESS_SAVE_DECL(mp); 400 401 td = curthread; 402 p = td->td_proc; 403 rval = 0; 404 #ifdef KTRACE 405 if (KTRPOINT(td, KTR_CSW)) 406 ktrcsw(1, 0); 407 #endif 408 CV_ASSERT(cvp, mp, td); 409 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mp->mtx_object, 410 "Waiting on \"%s\"", cvp->cv_description); 411 WITNESS_SAVE(&mp->mtx_object, mp); 412 413 if (cold || panicstr) { 414 /* 415 * After a panic, or during autoconfiguration, just give 416 * interrupts a chance, then just return; don't run any other 417 * thread or panic below, in case this is the idle process and 418 * already asleep. 419 */ 420 return 0; 421 } 422 423 mtx_lock_spin(&sched_lock); 424 425 CV_WAIT_VALIDATE(cvp, mp); 426 427 DROP_GIANT(); 428 mtx_unlock(mp); 429 430 cv_waitq_add(cvp, td); 431 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td); 432 sig = cv_switch_catch(td); 433 434 if (td->td_flags & TDF_TIMEOUT) { 435 td->td_flags &= ~TDF_TIMEOUT; 436 rval = EWOULDBLOCK; 437 } else if (td->td_flags & TDF_TIMOFAIL) 438 td->td_flags &= ~TDF_TIMOFAIL; 439 else if (callout_stop(&td->td_slpcallout) == 0) { 440 /* 441 * Work around race with cv_timedwait_end similar to that 442 * between msleep and endtsleep. 443 * Go back to sleep. 444 */ 445 TD_SET_SLEEPING(td); 446 td->td_proc->p_stats->p_ru.ru_nivcsw++; 447 mi_switch(); 448 td->td_flags &= ~TDF_TIMOFAIL; 449 } 450 mtx_unlock_spin(&sched_lock); 451 452 PROC_LOCK(p); 453 mtx_lock(&p->p_sigacts->ps_mtx); 454 if (sig == 0) 455 sig = cursig(td); 456 if (sig != 0) { 457 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig)) 458 rval = EINTR; 459 else 460 rval = ERESTART; 461 } 462 mtx_unlock(&p->p_sigacts->ps_mtx); 463 if (p->p_flag & P_WEXIT) 464 rval = EINTR; 465 PROC_UNLOCK(p); 466 467 #ifdef KTRACE 468 if (KTRPOINT(td, KTR_CSW)) 469 ktrcsw(0, 0); 470 #endif 471 PICKUP_GIANT(); 472 mtx_lock(mp); 473 WITNESS_RESTORE(&mp->mtx_object, mp); 474 475 return (rval); 476 } 477 478 /* 479 * Common code for signal and broadcast. Assumes waitq is not empty. Must be 480 * called with sched_lock held. 481 */ 482 static __inline void 483 cv_wakeup(struct cv *cvp) 484 { 485 struct thread *td; 486 487 mtx_assert(&sched_lock, MA_OWNED); 488 td = TAILQ_FIRST(&cvp->cv_waitq); 489 KASSERT(td->td_wchan == cvp, ("%s: bogus wchan", __func__)); 490 KASSERT(td->td_flags & TDF_CVWAITQ, ("%s: not on waitq", __func__)); 491 cv_waitq_remove(td); 492 TD_CLR_SLEEPING(td); 493 setrunnable(td); 494 } 495 496 /* 497 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 498 * the swapper if the process is not in memory, so that it can bring the 499 * sleeping process in. Note that this may also result in additional threads 500 * being made runnable. Should be called with the same mutex as was passed to 501 * cv_wait held. 502 */ 503 void 504 cv_signal(struct cv *cvp) 505 { 506 507 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__)); 508 mtx_lock_spin(&sched_lock); 509 if (!TAILQ_EMPTY(&cvp->cv_waitq)) { 510 CV_SIGNAL_VALIDATE(cvp); 511 cv_wakeup(cvp); 512 } 513 mtx_unlock_spin(&sched_lock); 514 } 515 516 /* 517 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 518 * Should be called with the same mutex as was passed to cv_wait held. 519 */ 520 void 521 cv_broadcast(struct cv *cvp) 522 { 523 524 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__)); 525 mtx_lock_spin(&sched_lock); 526 CV_SIGNAL_VALIDATE(cvp); 527 while (!TAILQ_EMPTY(&cvp->cv_waitq)) 528 cv_wakeup(cvp); 529 mtx_unlock_spin(&sched_lock); 530 } 531 532 /* 533 * Remove a thread from the wait queue of its condition variable. This may be 534 * called externally. 535 */ 536 void 537 cv_waitq_remove(struct thread *td) 538 { 539 struct cv *cvp; 540 541 mtx_assert(&sched_lock, MA_OWNED); 542 if ((cvp = td->td_wchan) != NULL && td->td_flags & TDF_CVWAITQ) { 543 TAILQ_REMOVE(&cvp->cv_waitq, td, td_slpq); 544 td->td_flags &= ~TDF_CVWAITQ; 545 td->td_wmesg = NULL; 546 TD_CLR_ON_SLEEPQ(td); 547 } 548 } 549 550 /* 551 * Timeout function for cv_timedwait. Put the thread on the runqueue and set 552 * its timeout flag. 553 */ 554 static void 555 cv_timedwait_end(void *arg) 556 { 557 struct thread *td; 558 559 td = arg; 560 CTR3(KTR_PROC, "cv_timedwait_end: thread %p (pid %d, %s)", 561 td, td->td_proc->p_pid, td->td_proc->p_comm); 562 mtx_lock_spin(&sched_lock); 563 if (TD_ON_SLEEPQ(td)) { 564 cv_waitq_remove(td); 565 td->td_flags |= TDF_TIMEOUT; 566 } else { 567 td->td_flags |= TDF_TIMOFAIL; 568 } 569 TD_CLR_SLEEPING(td); 570 setrunnable(td); 571 mtx_unlock_spin(&sched_lock); 572 } 573 574 /* 575 * For now only abort interruptable waits. 576 * The others will have to either complete on their own or have a timeout. 577 */ 578 void 579 cv_abort(struct thread *td) 580 { 581 582 CTR3(KTR_PROC, "cv_abort: thread %p (pid %d, %s)", td, 583 td->td_proc->p_pid, td->td_proc->p_comm); 584 mtx_lock_spin(&sched_lock); 585 if ((td->td_flags & (TDF_SINTR|TDF_TIMEOUT)) == TDF_SINTR) { 586 if (TD_ON_SLEEPQ(td)) { 587 cv_waitq_remove(td); 588 } 589 TD_CLR_SLEEPING(td); 590 setrunnable(td); 591 } 592 mtx_unlock_spin(&sched_lock); 593 } 594
Cache object: 43569a9d0cd01c754194ba9a6056260f
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]