Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rwlock.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD: releng/11.0/sys/kern/kern_rwlock.c 303953 2016-08-11 09:28:49Z mjg $"); 33 34 #include "opt_ddb.h" 35 #include "opt_hwpmc_hooks.h" 36 #include "opt_no_adaptive_rwlocks.h" 37 38 #include <sys/param.h> 39 #include <sys/kdb.h> 40 #include <sys/ktr.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/proc.h> 45 #include <sys/rwlock.h> 46 #include <sys/sched.h> 47 #include <sys/smp.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 #include <sys/turnstile.h> 51 52 #include <machine/cpu.h> 53 54 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) 55 #define ADAPTIVE_RWLOCKS 56 #endif 57 58 #ifdef HWPMC_HOOKS 59 #include <sys/pmckern.h> 60 PMC_SOFT_DECLARE( , , lock, failed); 61 #endif 62 63 /* 64 * Return the rwlock address when the lock cookie address is provided. 65 * This functionality assumes that struct rwlock* have a member named rw_lock. 66 */ 67 #define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock)) 68 69 #ifdef DDB 70 #include <ddb/ddb.h> 71 72 static void db_show_rwlock(const struct lock_object *lock); 73 #endif 74 static void assert_rw(const struct lock_object *lock, int what); 75 static void lock_rw(struct lock_object *lock, uintptr_t how); 76 #ifdef KDTRACE_HOOKS 77 static int owner_rw(const struct lock_object *lock, struct thread **owner); 78 #endif 79 static uintptr_t unlock_rw(struct lock_object *lock); 80 81 struct lock_class lock_class_rw = { 82 .lc_name = "rw", 83 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, 84 .lc_assert = assert_rw, 85 #ifdef DDB 86 .lc_ddb_show = db_show_rwlock, 87 #endif 88 .lc_lock = lock_rw, 89 .lc_unlock = unlock_rw, 90 #ifdef KDTRACE_HOOKS 91 .lc_owner = owner_rw, 92 #endif 93 }; 94 95 #ifdef ADAPTIVE_RWLOCKS 96 static int rowner_retries = 10; 97 static int rowner_loops = 10000; 98 static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, 99 "rwlock debugging"); 100 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); 101 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); 102 103 static struct lock_delay_config rw_delay = { 104 .initial = 1000, 105 .step = 500, 106 .min = 100, 107 .max = 5000, 108 }; 109 110 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_initial, CTLFLAG_RW, &rw_delay.initial, 111 0, ""); 112 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_step, CTLFLAG_RW, &rw_delay.step, 113 0, ""); 114 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_min, CTLFLAG_RW, &rw_delay.min, 115 0, ""); 116 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max, 117 0, ""); 118 119 static void 120 rw_delay_sysinit(void *dummy) 121 { 122 123 rw_delay.initial = mp_ncpus * 25; 124 rw_delay.step = (mp_ncpus * 25) / 2; 125 rw_delay.min = mp_ncpus * 5; 126 rw_delay.max = mp_ncpus * 25 * 10; 127 } 128 LOCK_DELAY_SYSINIT(rw_delay_sysinit); 129 #endif 130 131 /* 132 * Return a pointer to the owning thread if the lock is write-locked or 133 * NULL if the lock is unlocked or read-locked. 134 */ 135 #define rw_wowner(rw) \ 136 ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ 137 (struct thread *)RW_OWNER((rw)->rw_lock)) 138 139 /* 140 * Returns if a write owner is recursed. Write ownership is not assured 141 * here and should be previously checked. 142 */ 143 #define rw_recursed(rw) ((rw)->rw_recurse != 0) 144 145 /* 146 * Return true if curthread helds the lock. 147 */ 148 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread) 149 150 /* 151 * Return a pointer to the owning thread for this lock who should receive 152 * any priority lent by threads that block on this lock. Currently this 153 * is identical to rw_wowner(). 154 */ 155 #define rw_owner(rw) rw_wowner(rw) 156 157 #ifndef INVARIANTS 158 #define __rw_assert(c, what, file, line) 159 #endif 160 161 void 162 assert_rw(const struct lock_object *lock, int what) 163 { 164 165 rw_assert((const struct rwlock *)lock, what); 166 } 167 168 void 169 lock_rw(struct lock_object *lock, uintptr_t how) 170 { 171 struct rwlock *rw; 172 173 rw = (struct rwlock *)lock; 174 if (how) 175 rw_rlock(rw); 176 else 177 rw_wlock(rw); 178 } 179 180 uintptr_t 181 unlock_rw(struct lock_object *lock) 182 { 183 struct rwlock *rw; 184 185 rw = (struct rwlock *)lock; 186 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); 187 if (rw->rw_lock & RW_LOCK_READ) { 188 rw_runlock(rw); 189 return (1); 190 } else { 191 rw_wunlock(rw); 192 return (0); 193 } 194 } 195 196 #ifdef KDTRACE_HOOKS 197 int 198 owner_rw(const struct lock_object *lock, struct thread **owner) 199 { 200 const struct rwlock *rw = (const struct rwlock *)lock; 201 uintptr_t x = rw->rw_lock; 202 203 *owner = rw_wowner(rw); 204 return ((x & RW_LOCK_READ) != 0 ? (RW_READERS(x) != 0) : 205 (*owner != NULL)); 206 } 207 #endif 208 209 void 210 _rw_init_flags(volatile uintptr_t *c, const char *name, int opts) 211 { 212 struct rwlock *rw; 213 int flags; 214 215 rw = rwlock2rw(c); 216 217 MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET | 218 RW_RECURSE | RW_NEW)) == 0); 219 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, 220 ("%s: rw_lock not aligned for %s: %p", __func__, name, 221 &rw->rw_lock)); 222 223 flags = LO_UPGRADABLE; 224 if (opts & RW_DUPOK) 225 flags |= LO_DUPOK; 226 if (opts & RW_NOPROFILE) 227 flags |= LO_NOPROFILE; 228 if (!(opts & RW_NOWITNESS)) 229 flags |= LO_WITNESS; 230 if (opts & RW_RECURSE) 231 flags |= LO_RECURSABLE; 232 if (opts & RW_QUIET) 233 flags |= LO_QUIET; 234 if (opts & RW_NEW) 235 flags |= LO_NEW; 236 237 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); 238 rw->rw_lock = RW_UNLOCKED; 239 rw->rw_recurse = 0; 240 } 241 242 void 243 _rw_destroy(volatile uintptr_t *c) 244 { 245 struct rwlock *rw; 246 247 rw = rwlock2rw(c); 248 249 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); 250 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); 251 rw->rw_lock = RW_DESTROYED; 252 lock_destroy(&rw->lock_object); 253 } 254 255 void 256 rw_sysinit(void *arg) 257 { 258 struct rw_args *args = arg; 259 260 rw_init((struct rwlock *)args->ra_rw, args->ra_desc); 261 } 262 263 void 264 rw_sysinit_flags(void *arg) 265 { 266 struct rw_args_flags *args = arg; 267 268 rw_init_flags((struct rwlock *)args->ra_rw, args->ra_desc, 269 args->ra_flags); 270 } 271 272 int 273 _rw_wowned(const volatile uintptr_t *c) 274 { 275 276 return (rw_wowner(rwlock2rw(c)) == curthread); 277 } 278 279 void 280 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) 281 { 282 struct rwlock *rw; 283 284 if (SCHEDULER_STOPPED()) 285 return; 286 287 rw = rwlock2rw(c); 288 289 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 290 ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d", 291 curthread, rw->lock_object.lo_name, file, line)); 292 KASSERT(rw->rw_lock != RW_DESTROYED, 293 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 294 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 295 line, NULL); 296 __rw_wlock(rw, curthread, file, line); 297 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 298 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 299 TD_LOCKS_INC(curthread); 300 } 301 302 int 303 __rw_try_wlock(volatile uintptr_t *c, const char *file, int line) 304 { 305 struct rwlock *rw; 306 int rval; 307 308 if (SCHEDULER_STOPPED()) 309 return (1); 310 311 rw = rwlock2rw(c); 312 313 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 314 ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d", 315 curthread, rw->lock_object.lo_name, file, line)); 316 KASSERT(rw->rw_lock != RW_DESTROYED, 317 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 318 319 if (rw_wlocked(rw) && 320 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 321 rw->rw_recurse++; 322 rval = 1; 323 } else 324 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 325 (uintptr_t)curthread); 326 327 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 328 if (rval) { 329 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 330 file, line); 331 if (!rw_recursed(rw)) 332 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 333 rw, 0, 0, file, line, LOCKSTAT_WRITER); 334 TD_LOCKS_INC(curthread); 335 } 336 return (rval); 337 } 338 339 void 340 _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 341 { 342 struct rwlock *rw; 343 344 if (SCHEDULER_STOPPED()) 345 return; 346 347 rw = rwlock2rw(c); 348 349 KASSERT(rw->rw_lock != RW_DESTROYED, 350 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 351 __rw_assert(c, RA_WLOCKED, file, line); 352 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 353 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 354 line); 355 __rw_wunlock(rw, curthread, file, line); 356 TD_LOCKS_DEC(curthread); 357 } 358 359 /* 360 * Determines whether a new reader can acquire a lock. Succeeds if the 361 * reader already owns a read lock and the lock is locked for read to 362 * prevent deadlock from reader recursion. Also succeeds if the lock 363 * is unlocked and has no writer waiters or spinners. Failing otherwise 364 * prioritizes writers before readers. 365 */ 366 #define RW_CAN_READ(_rw) \ 367 ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) & \ 368 (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) == \ 369 RW_LOCK_READ) 370 371 void 372 __rw_rlock(volatile uintptr_t *c, const char *file, int line) 373 { 374 struct rwlock *rw; 375 struct turnstile *ts; 376 #ifdef ADAPTIVE_RWLOCKS 377 volatile struct thread *owner; 378 int spintries = 0; 379 int i; 380 #endif 381 #ifdef LOCK_PROFILING 382 uint64_t waittime = 0; 383 int contested = 0; 384 #endif 385 uintptr_t v; 386 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 387 struct lock_delay_arg lda; 388 #endif 389 #ifdef KDTRACE_HOOKS 390 uintptr_t state; 391 u_int sleep_cnt = 0; 392 int64_t sleep_time = 0; 393 int64_t all_time = 0; 394 #endif 395 396 if (SCHEDULER_STOPPED()) 397 return; 398 399 #if defined(ADAPTIVE_RWLOCKS) 400 lock_delay_arg_init(&lda, &rw_delay); 401 #elif defined(KDTRACE_HOOKS) 402 lock_delay_arg_init(&lda, NULL); 403 #endif 404 rw = rwlock2rw(c); 405 406 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 407 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 408 curthread, rw->lock_object.lo_name, file, line)); 409 KASSERT(rw->rw_lock != RW_DESTROYED, 410 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 411 KASSERT(rw_wowner(rw) != curthread, 412 ("rw_rlock: wlock already held for %s @ %s:%d", 413 rw->lock_object.lo_name, file, line)); 414 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 415 416 #ifdef KDTRACE_HOOKS 417 all_time -= lockstat_nsecs(&rw->lock_object); 418 state = rw->rw_lock; 419 #endif 420 for (;;) { 421 /* 422 * Handle the easy case. If no other thread has a write 423 * lock, then try to bump up the count of read locks. Note 424 * that we have to preserve the current state of the 425 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 426 * read lock, then rw_lock must have changed, so restart 427 * the loop. Note that this handles the case of a 428 * completely unlocked rwlock since such a lock is encoded 429 * as a read lock with no waiters. 430 */ 431 v = rw->rw_lock; 432 if (RW_CAN_READ(v)) { 433 /* 434 * The RW_LOCK_READ_WAITERS flag should only be set 435 * if the lock has been unlocked and write waiters 436 * were present. 437 */ 438 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, 439 v + RW_ONE_READER)) { 440 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 441 CTR4(KTR_LOCK, 442 "%s: %p succeed %p -> %p", __func__, 443 rw, (void *)v, 444 (void *)(v + RW_ONE_READER)); 445 break; 446 } 447 continue; 448 } 449 #ifdef KDTRACE_HOOKS 450 lda.spin_cnt++; 451 #endif 452 #ifdef HWPMC_HOOKS 453 PMC_SOFT_CALL( , , lock, failed); 454 #endif 455 lock_profile_obtain_lock_failed(&rw->lock_object, 456 &contested, &waittime); 457 458 #ifdef ADAPTIVE_RWLOCKS 459 /* 460 * If the owner is running on another CPU, spin until 461 * the owner stops running or the state of the lock 462 * changes. 463 */ 464 if ((v & RW_LOCK_READ) == 0) { 465 owner = (struct thread *)RW_OWNER(v); 466 if (TD_IS_RUNNING(owner)) { 467 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 468 CTR3(KTR_LOCK, 469 "%s: spinning on %p held by %p", 470 __func__, rw, owner); 471 KTR_STATE1(KTR_SCHED, "thread", 472 sched_tdname(curthread), "spinning", 473 "lockname:\"%s\"", rw->lock_object.lo_name); 474 while ((struct thread*)RW_OWNER(rw->rw_lock) == 475 owner && TD_IS_RUNNING(owner)) 476 lock_delay(&lda); 477 KTR_STATE0(KTR_SCHED, "thread", 478 sched_tdname(curthread), "running"); 479 continue; 480 } 481 } else if (spintries < rowner_retries) { 482 spintries++; 483 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 484 "spinning", "lockname:\"%s\"", 485 rw->lock_object.lo_name); 486 for (i = 0; i < rowner_loops; i++) { 487 v = rw->rw_lock; 488 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v)) 489 break; 490 cpu_spinwait(); 491 } 492 #ifdef KDTRACE_HOOKS 493 lda.spin_cnt += rowner_loops - i; 494 #endif 495 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 496 "running"); 497 if (i != rowner_loops) 498 continue; 499 } 500 #endif 501 502 /* 503 * Okay, now it's the hard case. Some other thread already 504 * has a write lock or there are write waiters present, 505 * acquire the turnstile lock so we can begin the process 506 * of blocking. 507 */ 508 ts = turnstile_trywait(&rw->lock_object); 509 510 /* 511 * The lock might have been released while we spun, so 512 * recheck its state and restart the loop if needed. 513 */ 514 v = rw->rw_lock; 515 if (RW_CAN_READ(v)) { 516 turnstile_cancel(ts); 517 continue; 518 } 519 520 #ifdef ADAPTIVE_RWLOCKS 521 /* 522 * The current lock owner might have started executing 523 * on another CPU (or the lock could have changed 524 * owners) while we were waiting on the turnstile 525 * chain lock. If so, drop the turnstile lock and try 526 * again. 527 */ 528 if ((v & RW_LOCK_READ) == 0) { 529 owner = (struct thread *)RW_OWNER(v); 530 if (TD_IS_RUNNING(owner)) { 531 turnstile_cancel(ts); 532 continue; 533 } 534 } 535 #endif 536 537 /* 538 * The lock is held in write mode or it already has waiters. 539 */ 540 MPASS(!RW_CAN_READ(v)); 541 542 /* 543 * If the RW_LOCK_READ_WAITERS flag is already set, then 544 * we can go ahead and block. If it is not set then try 545 * to set it. If we fail to set it drop the turnstile 546 * lock and restart the loop. 547 */ 548 if (!(v & RW_LOCK_READ_WAITERS)) { 549 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 550 v | RW_LOCK_READ_WAITERS)) { 551 turnstile_cancel(ts); 552 continue; 553 } 554 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 555 CTR2(KTR_LOCK, "%s: %p set read waiters flag", 556 __func__, rw); 557 } 558 559 /* 560 * We were unable to acquire the lock and the read waiters 561 * flag is set, so we must block on the turnstile. 562 */ 563 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 564 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 565 rw); 566 #ifdef KDTRACE_HOOKS 567 sleep_time -= lockstat_nsecs(&rw->lock_object); 568 #endif 569 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 570 #ifdef KDTRACE_HOOKS 571 sleep_time += lockstat_nsecs(&rw->lock_object); 572 sleep_cnt++; 573 #endif 574 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 575 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 576 __func__, rw); 577 } 578 #ifdef KDTRACE_HOOKS 579 all_time += lockstat_nsecs(&rw->lock_object); 580 if (sleep_time) 581 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 582 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 583 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 584 585 /* Record only the loops spinning and not sleeping. */ 586 if (lda.spin_cnt > sleep_cnt) 587 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 588 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 589 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 590 #endif 591 /* 592 * TODO: acquire "owner of record" here. Here be turnstile dragons 593 * however. turnstiles don't like owners changing between calls to 594 * turnstile_wait() currently. 595 */ 596 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 597 waittime, file, line, LOCKSTAT_READER); 598 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 599 WITNESS_LOCK(&rw->lock_object, 0, file, line); 600 TD_LOCKS_INC(curthread); 601 curthread->td_rw_rlocks++; 602 } 603 604 int 605 __rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 606 { 607 struct rwlock *rw; 608 uintptr_t x; 609 610 if (SCHEDULER_STOPPED()) 611 return (1); 612 613 rw = rwlock2rw(c); 614 615 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 616 ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d", 617 curthread, rw->lock_object.lo_name, file, line)); 618 619 for (;;) { 620 x = rw->rw_lock; 621 KASSERT(rw->rw_lock != RW_DESTROYED, 622 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 623 if (!(x & RW_LOCK_READ)) 624 break; 625 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 626 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 627 line); 628 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); 629 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 630 rw, 0, 0, file, line, LOCKSTAT_READER); 631 TD_LOCKS_INC(curthread); 632 curthread->td_rw_rlocks++; 633 return (1); 634 } 635 } 636 637 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 638 return (0); 639 } 640 641 void 642 _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) 643 { 644 struct rwlock *rw; 645 struct turnstile *ts; 646 uintptr_t x, v, queue; 647 648 if (SCHEDULER_STOPPED()) 649 return; 650 651 rw = rwlock2rw(c); 652 653 KASSERT(rw->rw_lock != RW_DESTROYED, 654 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 655 __rw_assert(c, RA_RLOCKED, file, line); 656 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 657 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 658 659 /* TODO: drop "owner of record" here. */ 660 661 for (;;) { 662 /* 663 * See if there is more than one read lock held. If so, 664 * just drop one and return. 665 */ 666 x = rw->rw_lock; 667 if (RW_READERS(x) > 1) { 668 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 669 x - RW_ONE_READER)) { 670 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 671 CTR4(KTR_LOCK, 672 "%s: %p succeeded %p -> %p", 673 __func__, rw, (void *)x, 674 (void *)(x - RW_ONE_READER)); 675 break; 676 } 677 continue; 678 } 679 /* 680 * If there aren't any waiters for a write lock, then try 681 * to drop it quickly. 682 */ 683 if (!(x & RW_LOCK_WAITERS)) { 684 MPASS((x & ~RW_LOCK_WRITE_SPINNER) == 685 RW_READERS_LOCK(1)); 686 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 687 RW_UNLOCKED)) { 688 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 689 CTR2(KTR_LOCK, "%s: %p last succeeded", 690 __func__, rw); 691 break; 692 } 693 continue; 694 } 695 /* 696 * Ok, we know we have waiters and we think we are the 697 * last reader, so grab the turnstile lock. 698 */ 699 turnstile_chain_lock(&rw->lock_object); 700 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 701 MPASS(v & RW_LOCK_WAITERS); 702 703 /* 704 * Try to drop our lock leaving the lock in a unlocked 705 * state. 706 * 707 * If you wanted to do explicit lock handoff you'd have to 708 * do it here. You'd also want to use turnstile_signal() 709 * and you'd have to handle the race where a higher 710 * priority thread blocks on the write lock before the 711 * thread you wakeup actually runs and have the new thread 712 * "steal" the lock. For now it's a lot simpler to just 713 * wakeup all of the waiters. 714 * 715 * As above, if we fail, then another thread might have 716 * acquired a read lock, so drop the turnstile lock and 717 * restart. 718 */ 719 x = RW_UNLOCKED; 720 if (v & RW_LOCK_WRITE_WAITERS) { 721 queue = TS_EXCLUSIVE_QUEUE; 722 x |= (v & RW_LOCK_READ_WAITERS); 723 } else 724 queue = TS_SHARED_QUEUE; 725 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, 726 x)) { 727 turnstile_chain_unlock(&rw->lock_object); 728 continue; 729 } 730 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 731 CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 732 __func__, rw); 733 734 /* 735 * Ok. The lock is released and all that's left is to 736 * wake up the waiters. Note that the lock might not be 737 * free anymore, but in that case the writers will just 738 * block again if they run before the new lock holder(s) 739 * release the lock. 740 */ 741 ts = turnstile_lookup(&rw->lock_object); 742 MPASS(ts != NULL); 743 turnstile_broadcast(ts, queue); 744 turnstile_unpend(ts, TS_SHARED_LOCK); 745 turnstile_chain_unlock(&rw->lock_object); 746 break; 747 } 748 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER); 749 TD_LOCKS_DEC(curthread); 750 curthread->td_rw_rlocks--; 751 } 752 753 /* 754 * This function is called when we are unable to obtain a write lock on the 755 * first try. This means that at least one other thread holds either a 756 * read or write lock. 757 */ 758 void 759 __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 760 int line) 761 { 762 struct rwlock *rw; 763 struct turnstile *ts; 764 #ifdef ADAPTIVE_RWLOCKS 765 volatile struct thread *owner; 766 int spintries = 0; 767 int i; 768 #endif 769 uintptr_t v, x; 770 #ifdef LOCK_PROFILING 771 uint64_t waittime = 0; 772 int contested = 0; 773 #endif 774 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 775 struct lock_delay_arg lda; 776 #endif 777 #ifdef KDTRACE_HOOKS 778 uintptr_t state; 779 u_int sleep_cnt = 0; 780 int64_t sleep_time = 0; 781 int64_t all_time = 0; 782 #endif 783 784 if (SCHEDULER_STOPPED()) 785 return; 786 787 #if defined(ADAPTIVE_RWLOCKS) 788 lock_delay_arg_init(&lda, &rw_delay); 789 #elif defined(KDTRACE_HOOKS) 790 lock_delay_arg_init(&lda, NULL); 791 #endif 792 rw = rwlock2rw(c); 793 794 if (rw_wlocked(rw)) { 795 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 796 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 797 __func__, rw->lock_object.lo_name, file, line)); 798 rw->rw_recurse++; 799 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 800 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 801 return; 802 } 803 804 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 805 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 806 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 807 808 #ifdef KDTRACE_HOOKS 809 all_time -= lockstat_nsecs(&rw->lock_object); 810 state = rw->rw_lock; 811 #endif 812 for (;;) { 813 if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid)) 814 break; 815 #ifdef KDTRACE_HOOKS 816 lda.spin_cnt++; 817 #endif 818 #ifdef HWPMC_HOOKS 819 PMC_SOFT_CALL( , , lock, failed); 820 #endif 821 lock_profile_obtain_lock_failed(&rw->lock_object, 822 &contested, &waittime); 823 #ifdef ADAPTIVE_RWLOCKS 824 /* 825 * If the lock is write locked and the owner is 826 * running on another CPU, spin until the owner stops 827 * running or the state of the lock changes. 828 */ 829 v = rw->rw_lock; 830 owner = (struct thread *)RW_OWNER(v); 831 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 832 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 833 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 834 __func__, rw, owner); 835 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 836 "spinning", "lockname:\"%s\"", 837 rw->lock_object.lo_name); 838 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 839 TD_IS_RUNNING(owner)) 840 lock_delay(&lda); 841 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 842 "running"); 843 continue; 844 } 845 if ((v & RW_LOCK_READ) && RW_READERS(v) && 846 spintries < rowner_retries) { 847 if (!(v & RW_LOCK_WRITE_SPINNER)) { 848 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 849 v | RW_LOCK_WRITE_SPINNER)) { 850 continue; 851 } 852 } 853 spintries++; 854 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 855 "spinning", "lockname:\"%s\"", 856 rw->lock_object.lo_name); 857 for (i = 0; i < rowner_loops; i++) { 858 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0) 859 break; 860 cpu_spinwait(); 861 } 862 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 863 "running"); 864 #ifdef KDTRACE_HOOKS 865 lda.spin_cnt += rowner_loops - i; 866 #endif 867 if (i != rowner_loops) 868 continue; 869 } 870 #endif 871 ts = turnstile_trywait(&rw->lock_object); 872 v = rw->rw_lock; 873 874 #ifdef ADAPTIVE_RWLOCKS 875 /* 876 * The current lock owner might have started executing 877 * on another CPU (or the lock could have changed 878 * owners) while we were waiting on the turnstile 879 * chain lock. If so, drop the turnstile lock and try 880 * again. 881 */ 882 if (!(v & RW_LOCK_READ)) { 883 owner = (struct thread *)RW_OWNER(v); 884 if (TD_IS_RUNNING(owner)) { 885 turnstile_cancel(ts); 886 continue; 887 } 888 } 889 #endif 890 /* 891 * Check for the waiters flags about this rwlock. 892 * If the lock was released, without maintain any pending 893 * waiters queue, simply try to acquire it. 894 * If a pending waiters queue is present, claim the lock 895 * ownership and maintain the pending queue. 896 */ 897 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 898 if ((v & ~x) == RW_UNLOCKED) { 899 x &= ~RW_LOCK_WRITE_SPINNER; 900 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { 901 if (x) 902 turnstile_claim(ts); 903 else 904 turnstile_cancel(ts); 905 break; 906 } 907 turnstile_cancel(ts); 908 continue; 909 } 910 /* 911 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 912 * set it. If we fail to set it, then loop back and try 913 * again. 914 */ 915 if (!(v & RW_LOCK_WRITE_WAITERS)) { 916 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 917 v | RW_LOCK_WRITE_WAITERS)) { 918 turnstile_cancel(ts); 919 continue; 920 } 921 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 922 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 923 __func__, rw); 924 } 925 /* 926 * We were unable to acquire the lock and the write waiters 927 * flag is set, so we must block on the turnstile. 928 */ 929 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 930 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 931 rw); 932 #ifdef KDTRACE_HOOKS 933 sleep_time -= lockstat_nsecs(&rw->lock_object); 934 #endif 935 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 936 #ifdef KDTRACE_HOOKS 937 sleep_time += lockstat_nsecs(&rw->lock_object); 938 sleep_cnt++; 939 #endif 940 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 941 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 942 __func__, rw); 943 #ifdef ADAPTIVE_RWLOCKS 944 spintries = 0; 945 #endif 946 } 947 #ifdef KDTRACE_HOOKS 948 all_time += lockstat_nsecs(&rw->lock_object); 949 if (sleep_time) 950 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 951 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 952 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 953 954 /* Record only the loops spinning and not sleeping. */ 955 if (lda.spin_cnt > sleep_cnt) 956 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 957 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 958 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 959 #endif 960 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 961 waittime, file, line, LOCKSTAT_WRITER); 962 } 963 964 /* 965 * This function is called if the first try at releasing a write lock failed. 966 * This means that one of the 2 waiter bits must be set indicating that at 967 * least one thread is waiting on this lock. 968 */ 969 void 970 __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 971 int line) 972 { 973 struct rwlock *rw; 974 struct turnstile *ts; 975 uintptr_t v; 976 int queue; 977 978 if (SCHEDULER_STOPPED()) 979 return; 980 981 rw = rwlock2rw(c); 982 983 if (rw_wlocked(rw) && rw_recursed(rw)) { 984 rw->rw_recurse--; 985 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 986 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 987 return; 988 } 989 990 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 991 ("%s: neither of the waiter flags are set", __func__)); 992 993 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 994 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 995 996 turnstile_chain_lock(&rw->lock_object); 997 ts = turnstile_lookup(&rw->lock_object); 998 MPASS(ts != NULL); 999 1000 /* 1001 * Use the same algo as sx locks for now. Prefer waking up shared 1002 * waiters if we have any over writers. This is probably not ideal. 1003 * 1004 * 'v' is the value we are going to write back to rw_lock. If we 1005 * have waiters on both queues, we need to preserve the state of 1006 * the waiter flag for the queue we don't wake up. For now this is 1007 * hardcoded for the algorithm mentioned above. 1008 * 1009 * In the case of both readers and writers waiting we wakeup the 1010 * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 1011 * new writer comes in before a reader it will claim the lock up 1012 * above. There is probably a potential priority inversion in 1013 * there that could be worked around either by waking both queues 1014 * of waiters or doing some complicated lock handoff gymnastics. 1015 */ 1016 v = RW_UNLOCKED; 1017 if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { 1018 queue = TS_EXCLUSIVE_QUEUE; 1019 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); 1020 } else 1021 queue = TS_SHARED_QUEUE; 1022 1023 /* Wake up all waiters for the specific queue. */ 1024 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1025 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 1026 queue == TS_SHARED_QUEUE ? "read" : "write"); 1027 turnstile_broadcast(ts, queue); 1028 atomic_store_rel_ptr(&rw->rw_lock, v); 1029 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1030 turnstile_chain_unlock(&rw->lock_object); 1031 } 1032 1033 /* 1034 * Attempt to do a non-blocking upgrade from a read lock to a write 1035 * lock. This will only succeed if this thread holds a single read 1036 * lock. Returns true if the upgrade succeeded and false otherwise. 1037 */ 1038 int 1039 __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line) 1040 { 1041 struct rwlock *rw; 1042 uintptr_t v, x, tid; 1043 struct turnstile *ts; 1044 int success; 1045 1046 if (SCHEDULER_STOPPED()) 1047 return (1); 1048 1049 rw = rwlock2rw(c); 1050 1051 KASSERT(rw->rw_lock != RW_DESTROYED, 1052 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 1053 __rw_assert(c, RA_RLOCKED, file, line); 1054 1055 /* 1056 * Attempt to switch from one reader to a writer. If there 1057 * are any write waiters, then we will have to lock the 1058 * turnstile first to prevent races with another writer 1059 * calling turnstile_wait() before we have claimed this 1060 * turnstile. So, do the simple case of no waiters first. 1061 */ 1062 tid = (uintptr_t)curthread; 1063 success = 0; 1064 for (;;) { 1065 v = rw->rw_lock; 1066 if (RW_READERS(v) > 1) 1067 break; 1068 if (!(v & RW_LOCK_WAITERS)) { 1069 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid); 1070 if (!success) 1071 continue; 1072 break; 1073 } 1074 1075 /* 1076 * Ok, we think we have waiters, so lock the turnstile. 1077 */ 1078 ts = turnstile_trywait(&rw->lock_object); 1079 v = rw->rw_lock; 1080 if (RW_READERS(v) > 1) { 1081 turnstile_cancel(ts); 1082 break; 1083 } 1084 /* 1085 * Try to switch from one reader to a writer again. This time 1086 * we honor the current state of the waiters flags. 1087 * If we obtain the lock with the flags set, then claim 1088 * ownership of the turnstile. 1089 */ 1090 x = rw->rw_lock & RW_LOCK_WAITERS; 1091 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); 1092 if (success) { 1093 if (x) 1094 turnstile_claim(ts); 1095 else 1096 turnstile_cancel(ts); 1097 break; 1098 } 1099 turnstile_cancel(ts); 1100 } 1101 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); 1102 if (success) { 1103 curthread->td_rw_rlocks--; 1104 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 1105 file, line); 1106 LOCKSTAT_RECORD0(rw__upgrade, rw); 1107 } 1108 return (success); 1109 } 1110 1111 /* 1112 * Downgrade a write lock into a single read lock. 1113 */ 1114 void 1115 __rw_downgrade(volatile uintptr_t *c, const char *file, int line) 1116 { 1117 struct rwlock *rw; 1118 struct turnstile *ts; 1119 uintptr_t tid, v; 1120 int rwait, wwait; 1121 1122 if (SCHEDULER_STOPPED()) 1123 return; 1124 1125 rw = rwlock2rw(c); 1126 1127 KASSERT(rw->rw_lock != RW_DESTROYED, 1128 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 1129 __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line); 1130 #ifndef INVARIANTS 1131 if (rw_recursed(rw)) 1132 panic("downgrade of a recursed lock"); 1133 #endif 1134 1135 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); 1136 1137 /* 1138 * Convert from a writer to a single reader. First we handle 1139 * the easy case with no waiters. If there are any waiters, we 1140 * lock the turnstile and "disown" the lock. 1141 */ 1142 tid = (uintptr_t)curthread; 1143 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) 1144 goto out; 1145 1146 /* 1147 * Ok, we think we have waiters, so lock the turnstile so we can 1148 * read the waiter flags without any races. 1149 */ 1150 turnstile_chain_lock(&rw->lock_object); 1151 v = rw->rw_lock & RW_LOCK_WAITERS; 1152 rwait = v & RW_LOCK_READ_WAITERS; 1153 wwait = v & RW_LOCK_WRITE_WAITERS; 1154 MPASS(rwait | wwait); 1155 1156 /* 1157 * Downgrade from a write lock while preserving waiters flag 1158 * and give up ownership of the turnstile. 1159 */ 1160 ts = turnstile_lookup(&rw->lock_object); 1161 MPASS(ts != NULL); 1162 if (!wwait) 1163 v &= ~RW_LOCK_READ_WAITERS; 1164 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); 1165 /* 1166 * Wake other readers if there are no writers pending. Otherwise they 1167 * won't be able to acquire the lock anyway. 1168 */ 1169 if (rwait && !wwait) { 1170 turnstile_broadcast(ts, TS_SHARED_QUEUE); 1171 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1172 } else 1173 turnstile_disown(ts); 1174 turnstile_chain_unlock(&rw->lock_object); 1175 out: 1176 curthread->td_rw_rlocks++; 1177 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); 1178 LOCKSTAT_RECORD0(rw__downgrade, rw); 1179 } 1180 1181 #ifdef INVARIANT_SUPPORT 1182 #ifndef INVARIANTS 1183 #undef __rw_assert 1184 #endif 1185 1186 /* 1187 * In the non-WITNESS case, rw_assert() can only detect that at least 1188 * *some* thread owns an rlock, but it cannot guarantee that *this* 1189 * thread owns an rlock. 1190 */ 1191 void 1192 __rw_assert(const volatile uintptr_t *c, int what, const char *file, int line) 1193 { 1194 const struct rwlock *rw; 1195 1196 if (panicstr != NULL) 1197 return; 1198 1199 rw = rwlock2rw(c); 1200 1201 switch (what) { 1202 case RA_LOCKED: 1203 case RA_LOCKED | RA_RECURSED: 1204 case RA_LOCKED | RA_NOTRECURSED: 1205 case RA_RLOCKED: 1206 case RA_RLOCKED | RA_RECURSED: 1207 case RA_RLOCKED | RA_NOTRECURSED: 1208 #ifdef WITNESS 1209 witness_assert(&rw->lock_object, what, file, line); 1210 #else 1211 /* 1212 * If some other thread has a write lock or we have one 1213 * and are asserting a read lock, fail. Also, if no one 1214 * has a lock at all, fail. 1215 */ 1216 if (rw->rw_lock == RW_UNLOCKED || 1217 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || 1218 rw_wowner(rw) != curthread))) 1219 panic("Lock %s not %slocked @ %s:%d\n", 1220 rw->lock_object.lo_name, (what & RA_RLOCKED) ? 1221 "read " : "", file, line); 1222 1223 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { 1224 if (rw_recursed(rw)) { 1225 if (what & RA_NOTRECURSED) 1226 panic("Lock %s recursed @ %s:%d\n", 1227 rw->lock_object.lo_name, file, 1228 line); 1229 } else if (what & RA_RECURSED) 1230 panic("Lock %s not recursed @ %s:%d\n", 1231 rw->lock_object.lo_name, file, line); 1232 } 1233 #endif 1234 break; 1235 case RA_WLOCKED: 1236 case RA_WLOCKED | RA_RECURSED: 1237 case RA_WLOCKED | RA_NOTRECURSED: 1238 if (rw_wowner(rw) != curthread) 1239 panic("Lock %s not exclusively locked @ %s:%d\n", 1240 rw->lock_object.lo_name, file, line); 1241 if (rw_recursed(rw)) { 1242 if (what & RA_NOTRECURSED) 1243 panic("Lock %s recursed @ %s:%d\n", 1244 rw->lock_object.lo_name, file, line); 1245 } else if (what & RA_RECURSED) 1246 panic("Lock %s not recursed @ %s:%d\n", 1247 rw->lock_object.lo_name, file, line); 1248 break; 1249 case RA_UNLOCKED: 1250 #ifdef WITNESS 1251 witness_assert(&rw->lock_object, what, file, line); 1252 #else 1253 /* 1254 * If we hold a write lock fail. We can't reliably check 1255 * to see if we hold a read lock or not. 1256 */ 1257 if (rw_wowner(rw) == curthread) 1258 panic("Lock %s exclusively locked @ %s:%d\n", 1259 rw->lock_object.lo_name, file, line); 1260 #endif 1261 break; 1262 default: 1263 panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 1264 line); 1265 } 1266 } 1267 #endif /* INVARIANT_SUPPORT */ 1268 1269 #ifdef DDB 1270 void 1271 db_show_rwlock(const struct lock_object *lock) 1272 { 1273 const struct rwlock *rw; 1274 struct thread *td; 1275 1276 rw = (const struct rwlock *)lock; 1277 1278 db_printf(" state: "); 1279 if (rw->rw_lock == RW_UNLOCKED) 1280 db_printf("UNLOCKED\n"); 1281 else if (rw->rw_lock == RW_DESTROYED) { 1282 db_printf("DESTROYED\n"); 1283 return; 1284 } else if (rw->rw_lock & RW_LOCK_READ) 1285 db_printf("RLOCK: %ju locks\n", 1286 (uintmax_t)(RW_READERS(rw->rw_lock))); 1287 else { 1288 td = rw_wowner(rw); 1289 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1290 td->td_tid, td->td_proc->p_pid, td->td_name); 1291 if (rw_recursed(rw)) 1292 db_printf(" recursed: %u\n", rw->rw_recurse); 1293 } 1294 db_printf(" waiters: "); 1295 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 1296 case RW_LOCK_READ_WAITERS: 1297 db_printf("readers\n"); 1298 break; 1299 case RW_LOCK_WRITE_WAITERS: 1300 db_printf("writers\n"); 1301 break; 1302 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 1303 db_printf("readers and writers\n"); 1304 break; 1305 default: 1306 db_printf("none\n"); 1307 break; 1308 } 1309 } 1310 1311 #endif
Cache object: 6cf5d92a173a064699e883b5a483cf43
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]