Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_rwlock.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD: releng/10.2/sys/kern/kern_rwlock.c 285759 2015-07-21 17:16:37Z markj $"); 33 34 #include "opt_ddb.h" 35 #include "opt_hwpmc_hooks.h" 36 #include "opt_kdtrace.h" 37 #include "opt_no_adaptive_rwlocks.h" 38 39 #include <sys/param.h> 40 #include <sys/kdb.h> 41 #include <sys/ktr.h> 42 #include <sys/kernel.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/proc.h> 46 #include <sys/rwlock.h> 47 #include <sys/sched.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 #include <sys/turnstile.h> 51 52 #include <machine/cpu.h> 53 54 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) 55 #define ADAPTIVE_RWLOCKS 56 #endif 57 58 #ifdef HWPMC_HOOKS 59 #include <sys/pmckern.h> 60 PMC_SOFT_DECLARE( , , lock, failed); 61 #endif 62 63 /* 64 * Return the rwlock address when the lock cookie address is provided. 65 * This functionality assumes that struct rwlock* have a member named rw_lock. 66 */ 67 #define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock)) 68 69 #ifdef ADAPTIVE_RWLOCKS 70 static int rowner_retries = 10; 71 static int rowner_loops = 10000; 72 static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, 73 "rwlock debugging"); 74 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); 75 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); 76 #endif 77 78 #ifdef DDB 79 #include <ddb/ddb.h> 80 81 static void db_show_rwlock(const struct lock_object *lock); 82 #endif 83 static void assert_rw(const struct lock_object *lock, int what); 84 static void lock_rw(struct lock_object *lock, uintptr_t how); 85 #ifdef KDTRACE_HOOKS 86 static int owner_rw(const struct lock_object *lock, struct thread **owner); 87 #endif 88 static uintptr_t unlock_rw(struct lock_object *lock); 89 90 struct lock_class lock_class_rw = { 91 .lc_name = "rw", 92 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, 93 .lc_assert = assert_rw, 94 #ifdef DDB 95 .lc_ddb_show = db_show_rwlock, 96 #endif 97 .lc_lock = lock_rw, 98 .lc_unlock = unlock_rw, 99 #ifdef KDTRACE_HOOKS 100 .lc_owner = owner_rw, 101 #endif 102 }; 103 104 /* 105 * Return a pointer to the owning thread if the lock is write-locked or 106 * NULL if the lock is unlocked or read-locked. 107 */ 108 #define rw_wowner(rw) \ 109 ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ 110 (struct thread *)RW_OWNER((rw)->rw_lock)) 111 112 /* 113 * Returns if a write owner is recursed. Write ownership is not assured 114 * here and should be previously checked. 115 */ 116 #define rw_recursed(rw) ((rw)->rw_recurse != 0) 117 118 /* 119 * Return true if curthread helds the lock. 120 */ 121 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread) 122 123 /* 124 * Return a pointer to the owning thread for this lock who should receive 125 * any priority lent by threads that block on this lock. Currently this 126 * is identical to rw_wowner(). 127 */ 128 #define rw_owner(rw) rw_wowner(rw) 129 130 #ifndef INVARIANTS 131 #define __rw_assert(c, what, file, line) 132 #endif 133 134 void 135 assert_rw(const struct lock_object *lock, int what) 136 { 137 138 rw_assert((const struct rwlock *)lock, what); 139 } 140 141 void 142 lock_rw(struct lock_object *lock, uintptr_t how) 143 { 144 struct rwlock *rw; 145 146 rw = (struct rwlock *)lock; 147 if (how) 148 rw_rlock(rw); 149 else 150 rw_wlock(rw); 151 } 152 153 uintptr_t 154 unlock_rw(struct lock_object *lock) 155 { 156 struct rwlock *rw; 157 158 rw = (struct rwlock *)lock; 159 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); 160 if (rw->rw_lock & RW_LOCK_READ) { 161 rw_runlock(rw); 162 return (1); 163 } else { 164 rw_wunlock(rw); 165 return (0); 166 } 167 } 168 169 #ifdef KDTRACE_HOOKS 170 int 171 owner_rw(const struct lock_object *lock, struct thread **owner) 172 { 173 const struct rwlock *rw = (const struct rwlock *)lock; 174 uintptr_t x = rw->rw_lock; 175 176 *owner = rw_wowner(rw); 177 return ((x & RW_LOCK_READ) != 0 ? (RW_READERS(x) != 0) : 178 (*owner != NULL)); 179 } 180 #endif 181 182 void 183 _rw_init_flags(volatile uintptr_t *c, const char *name, int opts) 184 { 185 struct rwlock *rw; 186 int flags; 187 188 rw = rwlock2rw(c); 189 190 MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET | 191 RW_RECURSE)) == 0); 192 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, 193 ("%s: rw_lock not aligned for %s: %p", __func__, name, 194 &rw->rw_lock)); 195 196 flags = LO_UPGRADABLE; 197 if (opts & RW_DUPOK) 198 flags |= LO_DUPOK; 199 if (opts & RW_NOPROFILE) 200 flags |= LO_NOPROFILE; 201 if (!(opts & RW_NOWITNESS)) 202 flags |= LO_WITNESS; 203 if (opts & RW_RECURSE) 204 flags |= LO_RECURSABLE; 205 if (opts & RW_QUIET) 206 flags |= LO_QUIET; 207 208 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); 209 rw->rw_lock = RW_UNLOCKED; 210 rw->rw_recurse = 0; 211 } 212 213 void 214 _rw_destroy(volatile uintptr_t *c) 215 { 216 struct rwlock *rw; 217 218 rw = rwlock2rw(c); 219 220 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); 221 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); 222 rw->rw_lock = RW_DESTROYED; 223 lock_destroy(&rw->lock_object); 224 } 225 226 void 227 rw_sysinit(void *arg) 228 { 229 struct rw_args *args = arg; 230 231 rw_init((struct rwlock *)args->ra_rw, args->ra_desc); 232 } 233 234 void 235 rw_sysinit_flags(void *arg) 236 { 237 struct rw_args_flags *args = arg; 238 239 rw_init_flags((struct rwlock *)args->ra_rw, args->ra_desc, 240 args->ra_flags); 241 } 242 243 int 244 _rw_wowned(const volatile uintptr_t *c) 245 { 246 247 return (rw_wowner(rwlock2rw(c)) == curthread); 248 } 249 250 void 251 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) 252 { 253 struct rwlock *rw; 254 255 if (SCHEDULER_STOPPED()) 256 return; 257 258 rw = rwlock2rw(c); 259 260 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 261 ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d", 262 curthread, rw->lock_object.lo_name, file, line)); 263 KASSERT(rw->rw_lock != RW_DESTROYED, 264 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 265 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 266 line, NULL); 267 __rw_wlock(rw, curthread, file, line); 268 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 269 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 270 curthread->td_locks++; 271 } 272 273 int 274 __rw_try_wlock(volatile uintptr_t *c, const char *file, int line) 275 { 276 struct rwlock *rw; 277 int rval; 278 279 if (SCHEDULER_STOPPED()) 280 return (1); 281 282 rw = rwlock2rw(c); 283 284 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 285 ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d", 286 curthread, rw->lock_object.lo_name, file, line)); 287 KASSERT(rw->rw_lock != RW_DESTROYED, 288 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 289 290 if (rw_wlocked(rw) && 291 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 292 rw->rw_recurse++; 293 rval = 1; 294 } else 295 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 296 (uintptr_t)curthread); 297 298 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 299 if (rval) { 300 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 301 file, line); 302 if (!rw_recursed(rw)) 303 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, 304 rw, 0, 0, file, line); 305 curthread->td_locks++; 306 } 307 return (rval); 308 } 309 310 void 311 _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 312 { 313 struct rwlock *rw; 314 315 if (SCHEDULER_STOPPED()) 316 return; 317 318 rw = rwlock2rw(c); 319 320 KASSERT(rw->rw_lock != RW_DESTROYED, 321 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 322 __rw_assert(c, RA_WLOCKED, file, line); 323 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 324 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 325 line); 326 if (!rw_recursed(rw)) 327 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw); 328 __rw_wunlock(rw, curthread, file, line); 329 curthread->td_locks--; 330 } 331 /* 332 * Determines whether a new reader can acquire a lock. Succeeds if the 333 * reader already owns a read lock and the lock is locked for read to 334 * prevent deadlock from reader recursion. Also succeeds if the lock 335 * is unlocked and has no writer waiters or spinners. Failing otherwise 336 * prioritizes writers before readers. 337 */ 338 #define RW_CAN_READ(_rw) \ 339 ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) & \ 340 (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) == \ 341 RW_LOCK_READ) 342 343 void 344 __rw_rlock(volatile uintptr_t *c, const char *file, int line) 345 { 346 struct rwlock *rw; 347 struct turnstile *ts; 348 #ifdef ADAPTIVE_RWLOCKS 349 volatile struct thread *owner; 350 int spintries = 0; 351 int i; 352 #endif 353 #ifdef LOCK_PROFILING 354 uint64_t waittime = 0; 355 int contested = 0; 356 #endif 357 uintptr_t v; 358 #ifdef KDTRACE_HOOKS 359 uintptr_t state; 360 uint64_t spin_cnt = 0; 361 uint64_t sleep_cnt = 0; 362 int64_t sleep_time = 0; 363 int64_t all_time = 0; 364 #endif 365 366 if (SCHEDULER_STOPPED()) 367 return; 368 369 rw = rwlock2rw(c); 370 371 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 372 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 373 curthread, rw->lock_object.lo_name, file, line)); 374 KASSERT(rw->rw_lock != RW_DESTROYED, 375 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 376 KASSERT(rw_wowner(rw) != curthread, 377 ("rw_rlock: wlock already held for %s @ %s:%d", 378 rw->lock_object.lo_name, file, line)); 379 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 380 381 #ifdef KDTRACE_HOOKS 382 all_time -= lockstat_nsecs(&rw->lock_object); 383 state = rw->rw_lock; 384 #endif 385 for (;;) { 386 #ifdef KDTRACE_HOOKS 387 spin_cnt++; 388 #endif 389 /* 390 * Handle the easy case. If no other thread has a write 391 * lock, then try to bump up the count of read locks. Note 392 * that we have to preserve the current state of the 393 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 394 * read lock, then rw_lock must have changed, so restart 395 * the loop. Note that this handles the case of a 396 * completely unlocked rwlock since such a lock is encoded 397 * as a read lock with no waiters. 398 */ 399 v = rw->rw_lock; 400 if (RW_CAN_READ(v)) { 401 /* 402 * The RW_LOCK_READ_WAITERS flag should only be set 403 * if the lock has been unlocked and write waiters 404 * were present. 405 */ 406 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, 407 v + RW_ONE_READER)) { 408 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 409 CTR4(KTR_LOCK, 410 "%s: %p succeed %p -> %p", __func__, 411 rw, (void *)v, 412 (void *)(v + RW_ONE_READER)); 413 break; 414 } 415 continue; 416 } 417 #ifdef HWPMC_HOOKS 418 PMC_SOFT_CALL( , , lock, failed); 419 #endif 420 lock_profile_obtain_lock_failed(&rw->lock_object, 421 &contested, &waittime); 422 423 #ifdef ADAPTIVE_RWLOCKS 424 /* 425 * If the owner is running on another CPU, spin until 426 * the owner stops running or the state of the lock 427 * changes. 428 */ 429 if ((v & RW_LOCK_READ) == 0) { 430 owner = (struct thread *)RW_OWNER(v); 431 if (TD_IS_RUNNING(owner)) { 432 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 433 CTR3(KTR_LOCK, 434 "%s: spinning on %p held by %p", 435 __func__, rw, owner); 436 KTR_STATE1(KTR_SCHED, "thread", 437 sched_tdname(curthread), "spinning", 438 "lockname:\"%s\"", rw->lock_object.lo_name); 439 while ((struct thread*)RW_OWNER(rw->rw_lock) == 440 owner && TD_IS_RUNNING(owner)) { 441 cpu_spinwait(); 442 #ifdef KDTRACE_HOOKS 443 spin_cnt++; 444 #endif 445 } 446 KTR_STATE0(KTR_SCHED, "thread", 447 sched_tdname(curthread), "running"); 448 continue; 449 } 450 } else if (spintries < rowner_retries) { 451 spintries++; 452 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 453 "spinning", "lockname:\"%s\"", 454 rw->lock_object.lo_name); 455 for (i = 0; i < rowner_loops; i++) { 456 v = rw->rw_lock; 457 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v)) 458 break; 459 cpu_spinwait(); 460 } 461 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 462 "running"); 463 if (i != rowner_loops) 464 continue; 465 } 466 #endif 467 468 /* 469 * Okay, now it's the hard case. Some other thread already 470 * has a write lock or there are write waiters present, 471 * acquire the turnstile lock so we can begin the process 472 * of blocking. 473 */ 474 ts = turnstile_trywait(&rw->lock_object); 475 476 /* 477 * The lock might have been released while we spun, so 478 * recheck its state and restart the loop if needed. 479 */ 480 v = rw->rw_lock; 481 if (RW_CAN_READ(v)) { 482 turnstile_cancel(ts); 483 continue; 484 } 485 486 #ifdef ADAPTIVE_RWLOCKS 487 /* 488 * The current lock owner might have started executing 489 * on another CPU (or the lock could have changed 490 * owners) while we were waiting on the turnstile 491 * chain lock. If so, drop the turnstile lock and try 492 * again. 493 */ 494 if ((v & RW_LOCK_READ) == 0) { 495 owner = (struct thread *)RW_OWNER(v); 496 if (TD_IS_RUNNING(owner)) { 497 turnstile_cancel(ts); 498 continue; 499 } 500 } 501 #endif 502 503 /* 504 * The lock is held in write mode or it already has waiters. 505 */ 506 MPASS(!RW_CAN_READ(v)); 507 508 /* 509 * If the RW_LOCK_READ_WAITERS flag is already set, then 510 * we can go ahead and block. If it is not set then try 511 * to set it. If we fail to set it drop the turnstile 512 * lock and restart the loop. 513 */ 514 if (!(v & RW_LOCK_READ_WAITERS)) { 515 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 516 v | RW_LOCK_READ_WAITERS)) { 517 turnstile_cancel(ts); 518 continue; 519 } 520 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 521 CTR2(KTR_LOCK, "%s: %p set read waiters flag", 522 __func__, rw); 523 } 524 525 /* 526 * We were unable to acquire the lock and the read waiters 527 * flag is set, so we must block on the turnstile. 528 */ 529 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 530 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 531 rw); 532 #ifdef KDTRACE_HOOKS 533 sleep_time -= lockstat_nsecs(&rw->lock_object); 534 #endif 535 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 536 #ifdef KDTRACE_HOOKS 537 sleep_time += lockstat_nsecs(&rw->lock_object); 538 sleep_cnt++; 539 #endif 540 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 541 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 542 __func__, rw); 543 } 544 #ifdef KDTRACE_HOOKS 545 all_time += lockstat_nsecs(&rw->lock_object); 546 if (sleep_time) 547 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, 548 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 549 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 550 551 /* Record only the loops spinning and not sleeping. */ 552 if (spin_cnt > sleep_cnt) 553 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time, 554 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 555 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 556 #endif 557 /* 558 * TODO: acquire "owner of record" here. Here be turnstile dragons 559 * however. turnstiles don't like owners changing between calls to 560 * turnstile_wait() currently. 561 */ 562 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, 563 waittime, file, line); 564 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 565 WITNESS_LOCK(&rw->lock_object, 0, file, line); 566 curthread->td_locks++; 567 curthread->td_rw_rlocks++; 568 } 569 570 int 571 __rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 572 { 573 struct rwlock *rw; 574 uintptr_t x; 575 576 if (SCHEDULER_STOPPED()) 577 return (1); 578 579 rw = rwlock2rw(c); 580 581 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 582 ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d", 583 curthread, rw->lock_object.lo_name, file, line)); 584 585 for (;;) { 586 x = rw->rw_lock; 587 KASSERT(rw->rw_lock != RW_DESTROYED, 588 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 589 if (!(x & RW_LOCK_READ)) 590 break; 591 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 592 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 593 line); 594 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); 595 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, 596 rw, 0, 0, file, line); 597 curthread->td_locks++; 598 curthread->td_rw_rlocks++; 599 return (1); 600 } 601 } 602 603 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 604 return (0); 605 } 606 607 void 608 _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) 609 { 610 struct rwlock *rw; 611 struct turnstile *ts; 612 uintptr_t x, v, queue; 613 614 if (SCHEDULER_STOPPED()) 615 return; 616 617 rw = rwlock2rw(c); 618 619 KASSERT(rw->rw_lock != RW_DESTROYED, 620 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 621 __rw_assert(c, RA_RLOCKED, file, line); 622 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 623 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 624 625 /* TODO: drop "owner of record" here. */ 626 627 for (;;) { 628 /* 629 * See if there is more than one read lock held. If so, 630 * just drop one and return. 631 */ 632 x = rw->rw_lock; 633 if (RW_READERS(x) > 1) { 634 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 635 x - RW_ONE_READER)) { 636 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 637 CTR4(KTR_LOCK, 638 "%s: %p succeeded %p -> %p", 639 __func__, rw, (void *)x, 640 (void *)(x - RW_ONE_READER)); 641 break; 642 } 643 continue; 644 } 645 /* 646 * If there aren't any waiters for a write lock, then try 647 * to drop it quickly. 648 */ 649 if (!(x & RW_LOCK_WAITERS)) { 650 MPASS((x & ~RW_LOCK_WRITE_SPINNER) == 651 RW_READERS_LOCK(1)); 652 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 653 RW_UNLOCKED)) { 654 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 655 CTR2(KTR_LOCK, "%s: %p last succeeded", 656 __func__, rw); 657 break; 658 } 659 continue; 660 } 661 /* 662 * Ok, we know we have waiters and we think we are the 663 * last reader, so grab the turnstile lock. 664 */ 665 turnstile_chain_lock(&rw->lock_object); 666 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 667 MPASS(v & RW_LOCK_WAITERS); 668 669 /* 670 * Try to drop our lock leaving the lock in a unlocked 671 * state. 672 * 673 * If you wanted to do explicit lock handoff you'd have to 674 * do it here. You'd also want to use turnstile_signal() 675 * and you'd have to handle the race where a higher 676 * priority thread blocks on the write lock before the 677 * thread you wakeup actually runs and have the new thread 678 * "steal" the lock. For now it's a lot simpler to just 679 * wakeup all of the waiters. 680 * 681 * As above, if we fail, then another thread might have 682 * acquired a read lock, so drop the turnstile lock and 683 * restart. 684 */ 685 x = RW_UNLOCKED; 686 if (v & RW_LOCK_WRITE_WAITERS) { 687 queue = TS_EXCLUSIVE_QUEUE; 688 x |= (v & RW_LOCK_READ_WAITERS); 689 } else 690 queue = TS_SHARED_QUEUE; 691 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, 692 x)) { 693 turnstile_chain_unlock(&rw->lock_object); 694 continue; 695 } 696 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 697 CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 698 __func__, rw); 699 700 /* 701 * Ok. The lock is released and all that's left is to 702 * wake up the waiters. Note that the lock might not be 703 * free anymore, but in that case the writers will just 704 * block again if they run before the new lock holder(s) 705 * release the lock. 706 */ 707 ts = turnstile_lookup(&rw->lock_object); 708 MPASS(ts != NULL); 709 turnstile_broadcast(ts, queue); 710 turnstile_unpend(ts, TS_SHARED_LOCK); 711 turnstile_chain_unlock(&rw->lock_object); 712 break; 713 } 714 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw); 715 curthread->td_locks--; 716 curthread->td_rw_rlocks--; 717 } 718 719 /* 720 * This function is called when we are unable to obtain a write lock on the 721 * first try. This means that at least one other thread holds either a 722 * read or write lock. 723 */ 724 void 725 __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 726 int line) 727 { 728 struct rwlock *rw; 729 struct turnstile *ts; 730 #ifdef ADAPTIVE_RWLOCKS 731 volatile struct thread *owner; 732 int spintries = 0; 733 int i; 734 #endif 735 uintptr_t v, x; 736 #ifdef LOCK_PROFILING 737 uint64_t waittime = 0; 738 int contested = 0; 739 #endif 740 #ifdef KDTRACE_HOOKS 741 uintptr_t state; 742 uint64_t spin_cnt = 0; 743 uint64_t sleep_cnt = 0; 744 int64_t sleep_time = 0; 745 int64_t all_time = 0; 746 #endif 747 748 if (SCHEDULER_STOPPED()) 749 return; 750 751 rw = rwlock2rw(c); 752 753 if (rw_wlocked(rw)) { 754 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 755 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 756 __func__, rw->lock_object.lo_name, file, line)); 757 rw->rw_recurse++; 758 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 759 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 760 return; 761 } 762 763 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 764 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 765 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 766 767 #ifdef KDTRACE_HOOKS 768 all_time -= lockstat_nsecs(&rw->lock_object); 769 state = rw->rw_lock; 770 #endif 771 while (!_rw_write_lock(rw, tid)) { 772 #ifdef KDTRACE_HOOKS 773 spin_cnt++; 774 #endif 775 #ifdef HWPMC_HOOKS 776 PMC_SOFT_CALL( , , lock, failed); 777 #endif 778 lock_profile_obtain_lock_failed(&rw->lock_object, 779 &contested, &waittime); 780 #ifdef ADAPTIVE_RWLOCKS 781 /* 782 * If the lock is write locked and the owner is 783 * running on another CPU, spin until the owner stops 784 * running or the state of the lock changes. 785 */ 786 v = rw->rw_lock; 787 owner = (struct thread *)RW_OWNER(v); 788 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 789 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 790 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 791 __func__, rw, owner); 792 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 793 "spinning", "lockname:\"%s\"", 794 rw->lock_object.lo_name); 795 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 796 TD_IS_RUNNING(owner)) { 797 cpu_spinwait(); 798 #ifdef KDTRACE_HOOKS 799 spin_cnt++; 800 #endif 801 } 802 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 803 "running"); 804 continue; 805 } 806 if ((v & RW_LOCK_READ) && RW_READERS(v) && 807 spintries < rowner_retries) { 808 if (!(v & RW_LOCK_WRITE_SPINNER)) { 809 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 810 v | RW_LOCK_WRITE_SPINNER)) { 811 continue; 812 } 813 } 814 spintries++; 815 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 816 "spinning", "lockname:\"%s\"", 817 rw->lock_object.lo_name); 818 for (i = 0; i < rowner_loops; i++) { 819 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0) 820 break; 821 cpu_spinwait(); 822 } 823 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 824 "running"); 825 #ifdef KDTRACE_HOOKS 826 spin_cnt += rowner_loops - i; 827 #endif 828 if (i != rowner_loops) 829 continue; 830 } 831 #endif 832 ts = turnstile_trywait(&rw->lock_object); 833 v = rw->rw_lock; 834 835 #ifdef ADAPTIVE_RWLOCKS 836 /* 837 * The current lock owner might have started executing 838 * on another CPU (or the lock could have changed 839 * owners) while we were waiting on the turnstile 840 * chain lock. If so, drop the turnstile lock and try 841 * again. 842 */ 843 if (!(v & RW_LOCK_READ)) { 844 owner = (struct thread *)RW_OWNER(v); 845 if (TD_IS_RUNNING(owner)) { 846 turnstile_cancel(ts); 847 continue; 848 } 849 } 850 #endif 851 /* 852 * Check for the waiters flags about this rwlock. 853 * If the lock was released, without maintain any pending 854 * waiters queue, simply try to acquire it. 855 * If a pending waiters queue is present, claim the lock 856 * ownership and maintain the pending queue. 857 */ 858 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 859 if ((v & ~x) == RW_UNLOCKED) { 860 x &= ~RW_LOCK_WRITE_SPINNER; 861 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { 862 if (x) 863 turnstile_claim(ts); 864 else 865 turnstile_cancel(ts); 866 break; 867 } 868 turnstile_cancel(ts); 869 continue; 870 } 871 /* 872 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 873 * set it. If we fail to set it, then loop back and try 874 * again. 875 */ 876 if (!(v & RW_LOCK_WRITE_WAITERS)) { 877 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 878 v | RW_LOCK_WRITE_WAITERS)) { 879 turnstile_cancel(ts); 880 continue; 881 } 882 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 883 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 884 __func__, rw); 885 } 886 /* 887 * We were unable to acquire the lock and the write waiters 888 * flag is set, so we must block on the turnstile. 889 */ 890 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 891 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 892 rw); 893 #ifdef KDTRACE_HOOKS 894 sleep_time -= lockstat_nsecs(&rw->lock_object); 895 #endif 896 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 897 #ifdef KDTRACE_HOOKS 898 sleep_time += lockstat_nsecs(&rw->lock_object); 899 sleep_cnt++; 900 #endif 901 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 902 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 903 __func__, rw); 904 #ifdef ADAPTIVE_RWLOCKS 905 spintries = 0; 906 #endif 907 } 908 #ifdef KDTRACE_HOOKS 909 all_time += lockstat_nsecs(&rw->lock_object); 910 if (sleep_time) 911 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, 912 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 913 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 914 915 /* Record only the loops spinning and not sleeping. */ 916 if (spin_cnt > sleep_cnt) 917 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time, 918 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 919 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 920 #endif 921 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, 922 waittime, file, line); 923 } 924 925 /* 926 * This function is called if the first try at releasing a write lock failed. 927 * This means that one of the 2 waiter bits must be set indicating that at 928 * least one thread is waiting on this lock. 929 */ 930 void 931 __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 932 int line) 933 { 934 struct rwlock *rw; 935 struct turnstile *ts; 936 uintptr_t v; 937 int queue; 938 939 if (SCHEDULER_STOPPED()) 940 return; 941 942 rw = rwlock2rw(c); 943 944 if (rw_wlocked(rw) && rw_recursed(rw)) { 945 rw->rw_recurse--; 946 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 947 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 948 return; 949 } 950 951 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 952 ("%s: neither of the waiter flags are set", __func__)); 953 954 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 955 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 956 957 turnstile_chain_lock(&rw->lock_object); 958 ts = turnstile_lookup(&rw->lock_object); 959 MPASS(ts != NULL); 960 961 /* 962 * Use the same algo as sx locks for now. Prefer waking up shared 963 * waiters if we have any over writers. This is probably not ideal. 964 * 965 * 'v' is the value we are going to write back to rw_lock. If we 966 * have waiters on both queues, we need to preserve the state of 967 * the waiter flag for the queue we don't wake up. For now this is 968 * hardcoded for the algorithm mentioned above. 969 * 970 * In the case of both readers and writers waiting we wakeup the 971 * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 972 * new writer comes in before a reader it will claim the lock up 973 * above. There is probably a potential priority inversion in 974 * there that could be worked around either by waking both queues 975 * of waiters or doing some complicated lock handoff gymnastics. 976 */ 977 v = RW_UNLOCKED; 978 if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { 979 queue = TS_EXCLUSIVE_QUEUE; 980 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); 981 } else 982 queue = TS_SHARED_QUEUE; 983 984 /* Wake up all waiters for the specific queue. */ 985 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 986 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 987 queue == TS_SHARED_QUEUE ? "read" : "write"); 988 turnstile_broadcast(ts, queue); 989 atomic_store_rel_ptr(&rw->rw_lock, v); 990 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 991 turnstile_chain_unlock(&rw->lock_object); 992 } 993 994 /* 995 * Attempt to do a non-blocking upgrade from a read lock to a write 996 * lock. This will only succeed if this thread holds a single read 997 * lock. Returns true if the upgrade succeeded and false otherwise. 998 */ 999 int 1000 __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line) 1001 { 1002 struct rwlock *rw; 1003 uintptr_t v, x, tid; 1004 struct turnstile *ts; 1005 int success; 1006 1007 if (SCHEDULER_STOPPED()) 1008 return (1); 1009 1010 rw = rwlock2rw(c); 1011 1012 KASSERT(rw->rw_lock != RW_DESTROYED, 1013 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 1014 __rw_assert(c, RA_RLOCKED, file, line); 1015 1016 /* 1017 * Attempt to switch from one reader to a writer. If there 1018 * are any write waiters, then we will have to lock the 1019 * turnstile first to prevent races with another writer 1020 * calling turnstile_wait() before we have claimed this 1021 * turnstile. So, do the simple case of no waiters first. 1022 */ 1023 tid = (uintptr_t)curthread; 1024 success = 0; 1025 for (;;) { 1026 v = rw->rw_lock; 1027 if (RW_READERS(v) > 1) 1028 break; 1029 if (!(v & RW_LOCK_WAITERS)) { 1030 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid); 1031 if (!success) 1032 continue; 1033 break; 1034 } 1035 1036 /* 1037 * Ok, we think we have waiters, so lock the turnstile. 1038 */ 1039 ts = turnstile_trywait(&rw->lock_object); 1040 v = rw->rw_lock; 1041 if (RW_READERS(v) > 1) { 1042 turnstile_cancel(ts); 1043 break; 1044 } 1045 /* 1046 * Try to switch from one reader to a writer again. This time 1047 * we honor the current state of the waiters flags. 1048 * If we obtain the lock with the flags set, then claim 1049 * ownership of the turnstile. 1050 */ 1051 x = rw->rw_lock & RW_LOCK_WAITERS; 1052 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); 1053 if (success) { 1054 if (x) 1055 turnstile_claim(ts); 1056 else 1057 turnstile_cancel(ts); 1058 break; 1059 } 1060 turnstile_cancel(ts); 1061 } 1062 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); 1063 if (success) { 1064 curthread->td_rw_rlocks--; 1065 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 1066 file, line); 1067 LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw); 1068 } 1069 return (success); 1070 } 1071 1072 /* 1073 * Downgrade a write lock into a single read lock. 1074 */ 1075 void 1076 __rw_downgrade(volatile uintptr_t *c, const char *file, int line) 1077 { 1078 struct rwlock *rw; 1079 struct turnstile *ts; 1080 uintptr_t tid, v; 1081 int rwait, wwait; 1082 1083 if (SCHEDULER_STOPPED()) 1084 return; 1085 1086 rw = rwlock2rw(c); 1087 1088 KASSERT(rw->rw_lock != RW_DESTROYED, 1089 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 1090 __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line); 1091 #ifndef INVARIANTS 1092 if (rw_recursed(rw)) 1093 panic("downgrade of a recursed lock"); 1094 #endif 1095 1096 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); 1097 1098 /* 1099 * Convert from a writer to a single reader. First we handle 1100 * the easy case with no waiters. If there are any waiters, we 1101 * lock the turnstile and "disown" the lock. 1102 */ 1103 tid = (uintptr_t)curthread; 1104 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) 1105 goto out; 1106 1107 /* 1108 * Ok, we think we have waiters, so lock the turnstile so we can 1109 * read the waiter flags without any races. 1110 */ 1111 turnstile_chain_lock(&rw->lock_object); 1112 v = rw->rw_lock & RW_LOCK_WAITERS; 1113 rwait = v & RW_LOCK_READ_WAITERS; 1114 wwait = v & RW_LOCK_WRITE_WAITERS; 1115 MPASS(rwait | wwait); 1116 1117 /* 1118 * Downgrade from a write lock while preserving waiters flag 1119 * and give up ownership of the turnstile. 1120 */ 1121 ts = turnstile_lookup(&rw->lock_object); 1122 MPASS(ts != NULL); 1123 if (!wwait) 1124 v &= ~RW_LOCK_READ_WAITERS; 1125 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); 1126 /* 1127 * Wake other readers if there are no writers pending. Otherwise they 1128 * won't be able to acquire the lock anyway. 1129 */ 1130 if (rwait && !wwait) { 1131 turnstile_broadcast(ts, TS_SHARED_QUEUE); 1132 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1133 } else 1134 turnstile_disown(ts); 1135 turnstile_chain_unlock(&rw->lock_object); 1136 out: 1137 curthread->td_rw_rlocks++; 1138 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); 1139 LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw); 1140 } 1141 1142 #ifdef INVARIANT_SUPPORT 1143 #ifndef INVARIANTS 1144 #undef __rw_assert 1145 #endif 1146 1147 /* 1148 * In the non-WITNESS case, rw_assert() can only detect that at least 1149 * *some* thread owns an rlock, but it cannot guarantee that *this* 1150 * thread owns an rlock. 1151 */ 1152 void 1153 __rw_assert(const volatile uintptr_t *c, int what, const char *file, int line) 1154 { 1155 const struct rwlock *rw; 1156 1157 if (panicstr != NULL) 1158 return; 1159 1160 rw = rwlock2rw(c); 1161 1162 switch (what) { 1163 case RA_LOCKED: 1164 case RA_LOCKED | RA_RECURSED: 1165 case RA_LOCKED | RA_NOTRECURSED: 1166 case RA_RLOCKED: 1167 case RA_RLOCKED | RA_RECURSED: 1168 case RA_RLOCKED | RA_NOTRECURSED: 1169 #ifdef WITNESS 1170 witness_assert(&rw->lock_object, what, file, line); 1171 #else 1172 /* 1173 * If some other thread has a write lock or we have one 1174 * and are asserting a read lock, fail. Also, if no one 1175 * has a lock at all, fail. 1176 */ 1177 if (rw->rw_lock == RW_UNLOCKED || 1178 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || 1179 rw_wowner(rw) != curthread))) 1180 panic("Lock %s not %slocked @ %s:%d\n", 1181 rw->lock_object.lo_name, (what & RA_RLOCKED) ? 1182 "read " : "", file, line); 1183 1184 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { 1185 if (rw_recursed(rw)) { 1186 if (what & RA_NOTRECURSED) 1187 panic("Lock %s recursed @ %s:%d\n", 1188 rw->lock_object.lo_name, file, 1189 line); 1190 } else if (what & RA_RECURSED) 1191 panic("Lock %s not recursed @ %s:%d\n", 1192 rw->lock_object.lo_name, file, line); 1193 } 1194 #endif 1195 break; 1196 case RA_WLOCKED: 1197 case RA_WLOCKED | RA_RECURSED: 1198 case RA_WLOCKED | RA_NOTRECURSED: 1199 if (rw_wowner(rw) != curthread) 1200 panic("Lock %s not exclusively locked @ %s:%d\n", 1201 rw->lock_object.lo_name, file, line); 1202 if (rw_recursed(rw)) { 1203 if (what & RA_NOTRECURSED) 1204 panic("Lock %s recursed @ %s:%d\n", 1205 rw->lock_object.lo_name, file, line); 1206 } else if (what & RA_RECURSED) 1207 panic("Lock %s not recursed @ %s:%d\n", 1208 rw->lock_object.lo_name, file, line); 1209 break; 1210 case RA_UNLOCKED: 1211 #ifdef WITNESS 1212 witness_assert(&rw->lock_object, what, file, line); 1213 #else 1214 /* 1215 * If we hold a write lock fail. We can't reliably check 1216 * to see if we hold a read lock or not. 1217 */ 1218 if (rw_wowner(rw) == curthread) 1219 panic("Lock %s exclusively locked @ %s:%d\n", 1220 rw->lock_object.lo_name, file, line); 1221 #endif 1222 break; 1223 default: 1224 panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 1225 line); 1226 } 1227 } 1228 #endif /* INVARIANT_SUPPORT */ 1229 1230 #ifdef DDB 1231 void 1232 db_show_rwlock(const struct lock_object *lock) 1233 { 1234 const struct rwlock *rw; 1235 struct thread *td; 1236 1237 rw = (const struct rwlock *)lock; 1238 1239 db_printf(" state: "); 1240 if (rw->rw_lock == RW_UNLOCKED) 1241 db_printf("UNLOCKED\n"); 1242 else if (rw->rw_lock == RW_DESTROYED) { 1243 db_printf("DESTROYED\n"); 1244 return; 1245 } else if (rw->rw_lock & RW_LOCK_READ) 1246 db_printf("RLOCK: %ju locks\n", 1247 (uintmax_t)(RW_READERS(rw->rw_lock))); 1248 else { 1249 td = rw_wowner(rw); 1250 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1251 td->td_tid, td->td_proc->p_pid, td->td_name); 1252 if (rw_recursed(rw)) 1253 db_printf(" recursed: %u\n", rw->rw_recurse); 1254 } 1255 db_printf(" waiters: "); 1256 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 1257 case RW_LOCK_READ_WAITERS: 1258 db_printf("readers\n"); 1259 break; 1260 case RW_LOCK_WRITE_WAITERS: 1261 db_printf("writers\n"); 1262 break; 1263 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 1264 db_printf("readers and writers\n"); 1265 break; 1266 default: 1267 db_printf("none\n"); 1268 break; 1269 } 1270 } 1271 1272 #endif
Cache object: cdf07ea75e31f14f839f9f558771857a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]