Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_sx.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice(s), this list of conditions and the following disclaimer as 11 * the first lines of this file unmodified other than the possible 12 * addition of one or more copyright notices. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice(s), this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27 * DAMAGE. 28 */ 29 30 /* 31 * Shared/exclusive locks. This implementation attempts to ensure 32 * deterministic lock granting behavior, so that slocks and xlocks are 33 * interleaved. 34 * 35 * Priority propagation will not generally raise the priority of lock holders, 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39 #include "opt_ddb.h" 40 #include "opt_hwpmc_hooks.h" 41 #include "opt_kdtrace.h" 42 #include "opt_no_adaptive_sx.h" 43 44 #include <sys/cdefs.h> 45 __FBSDID("$FreeBSD: releng/9.1/sys/kern/kern_sx.c 236238 2012-05-29 14:50:21Z fabient $"); 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/ktr.h> 50 #include <sys/lock.h> 51 #include <sys/mutex.h> 52 #include <sys/proc.h> 53 #include <sys/sleepqueue.h> 54 #include <sys/sx.h> 55 #include <sys/sysctl.h> 56 57 #if defined(SMP) && !defined(NO_ADAPTIVE_SX) 58 #include <machine/cpu.h> 59 #endif 60 61 #ifdef DDB 62 #include <ddb/ddb.h> 63 #endif 64 65 #if defined(SMP) && !defined(NO_ADAPTIVE_SX) 66 #define ADAPTIVE_SX 67 #endif 68 69 CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE); 70 71 #ifdef HWPMC_HOOKS 72 #include <sys/pmckern.h> 73 PMC_SOFT_DECLARE( , , lock, failed); 74 #endif 75 76 /* Handy macros for sleep queues. */ 77 #define SQ_EXCLUSIVE_QUEUE 0 78 #define SQ_SHARED_QUEUE 1 79 80 #ifdef ADAPTIVE_SX 81 #define ASX_RETRIES 10 82 #define ASX_LOOPS 10000 83 #endif 84 85 /* 86 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 87 * drop Giant anytime we have to sleep or if we adaptively spin. 88 */ 89 #define GIANT_DECLARE \ 90 int _giantcnt = 0; \ 91 WITNESS_SAVE_DECL(Giant) \ 92 93 #define GIANT_SAVE() do { \ 94 if (mtx_owned(&Giant)) { \ 95 WITNESS_SAVE(&Giant.lock_object, Giant); \ 96 while (mtx_owned(&Giant)) { \ 97 _giantcnt++; \ 98 mtx_unlock(&Giant); \ 99 } \ 100 } \ 101 } while (0) 102 103 #define GIANT_RESTORE() do { \ 104 if (_giantcnt > 0) { \ 105 mtx_assert(&Giant, MA_NOTOWNED); \ 106 while (_giantcnt--) \ 107 mtx_lock(&Giant); \ 108 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 109 } \ 110 } while (0) 111 112 /* 113 * Returns true if an exclusive lock is recursed. It assumes 114 * curthread currently has an exclusive lock. 115 */ 116 #define sx_recurse lock_object.lo_data 117 #define sx_recursed(sx) ((sx)->sx_recurse != 0) 118 119 static void assert_sx(struct lock_object *lock, int what); 120 #ifdef DDB 121 static void db_show_sx(struct lock_object *lock); 122 #endif 123 static void lock_sx(struct lock_object *lock, int how); 124 #ifdef KDTRACE_HOOKS 125 static int owner_sx(struct lock_object *lock, struct thread **owner); 126 #endif 127 static int unlock_sx(struct lock_object *lock); 128 129 struct lock_class lock_class_sx = { 130 .lc_name = "sx", 131 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 132 .lc_assert = assert_sx, 133 #ifdef DDB 134 .lc_ddb_show = db_show_sx, 135 #endif 136 .lc_lock = lock_sx, 137 .lc_unlock = unlock_sx, 138 #ifdef KDTRACE_HOOKS 139 .lc_owner = owner_sx, 140 #endif 141 }; 142 143 #ifndef INVARIANTS 144 #define _sx_assert(sx, what, file, line) 145 #endif 146 147 void 148 assert_sx(struct lock_object *lock, int what) 149 { 150 151 sx_assert((struct sx *)lock, what); 152 } 153 154 void 155 lock_sx(struct lock_object *lock, int how) 156 { 157 struct sx *sx; 158 159 sx = (struct sx *)lock; 160 if (how) 161 sx_xlock(sx); 162 else 163 sx_slock(sx); 164 } 165 166 int 167 unlock_sx(struct lock_object *lock) 168 { 169 struct sx *sx; 170 171 sx = (struct sx *)lock; 172 sx_assert(sx, SA_LOCKED | SA_NOTRECURSED); 173 if (sx_xlocked(sx)) { 174 sx_xunlock(sx); 175 return (1); 176 } else { 177 sx_sunlock(sx); 178 return (0); 179 } 180 } 181 182 #ifdef KDTRACE_HOOKS 183 int 184 owner_sx(struct lock_object *lock, struct thread **owner) 185 { 186 struct sx *sx = (struct sx *)lock; 187 uintptr_t x = sx->sx_lock; 188 189 *owner = (struct thread *)SX_OWNER(x); 190 return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) : 191 (*owner != NULL)); 192 } 193 #endif 194 195 void 196 sx_sysinit(void *arg) 197 { 198 struct sx_args *sargs = arg; 199 200 sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags); 201 } 202 203 void 204 sx_init_flags(struct sx *sx, const char *description, int opts) 205 { 206 int flags; 207 208 MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK | 209 SX_NOPROFILE | SX_NOADAPTIVE)) == 0); 210 ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock, 211 ("%s: sx_lock not aligned for %s: %p", __func__, description, 212 &sx->sx_lock)); 213 214 flags = LO_SLEEPABLE | LO_UPGRADABLE; 215 if (opts & SX_DUPOK) 216 flags |= LO_DUPOK; 217 if (opts & SX_NOPROFILE) 218 flags |= LO_NOPROFILE; 219 if (!(opts & SX_NOWITNESS)) 220 flags |= LO_WITNESS; 221 if (opts & SX_RECURSE) 222 flags |= LO_RECURSABLE; 223 if (opts & SX_QUIET) 224 flags |= LO_QUIET; 225 226 flags |= opts & SX_NOADAPTIVE; 227 sx->sx_lock = SX_LOCK_UNLOCKED; 228 sx->sx_recurse = 0; 229 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 230 } 231 232 void 233 sx_destroy(struct sx *sx) 234 { 235 236 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 237 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 238 sx->sx_lock = SX_LOCK_DESTROYED; 239 lock_destroy(&sx->lock_object); 240 } 241 242 int 243 _sx_slock(struct sx *sx, int opts, const char *file, int line) 244 { 245 int error = 0; 246 247 if (SCHEDULER_STOPPED()) 248 return (0); 249 MPASS(curthread != NULL); 250 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 251 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 252 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 253 error = __sx_slock(sx, opts, file, line); 254 if (!error) { 255 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 256 WITNESS_LOCK(&sx->lock_object, 0, file, line); 257 curthread->td_locks++; 258 } 259 260 return (error); 261 } 262 263 int 264 _sx_try_slock(struct sx *sx, const char *file, int line) 265 { 266 uintptr_t x; 267 268 if (SCHEDULER_STOPPED()) 269 return (1); 270 271 for (;;) { 272 x = sx->sx_lock; 273 KASSERT(x != SX_LOCK_DESTROYED, 274 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 275 if (!(x & SX_LOCK_SHARED)) 276 break; 277 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 278 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 279 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 280 curthread->td_locks++; 281 return (1); 282 } 283 } 284 285 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 286 return (0); 287 } 288 289 int 290 _sx_xlock(struct sx *sx, int opts, const char *file, int line) 291 { 292 int error = 0; 293 294 if (SCHEDULER_STOPPED()) 295 return (0); 296 MPASS(curthread != NULL); 297 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 298 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 299 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 300 line, NULL); 301 error = __sx_xlock(sx, curthread, opts, file, line); 302 if (!error) { 303 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, 304 file, line); 305 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 306 curthread->td_locks++; 307 } 308 309 return (error); 310 } 311 312 int 313 _sx_try_xlock(struct sx *sx, const char *file, int line) 314 { 315 int rval; 316 317 if (SCHEDULER_STOPPED()) 318 return (1); 319 320 MPASS(curthread != NULL); 321 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 322 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 323 324 if (sx_xlocked(sx) && 325 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) { 326 sx->sx_recurse++; 327 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 328 rval = 1; 329 } else 330 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 331 (uintptr_t)curthread); 332 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 333 if (rval) { 334 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 335 file, line); 336 curthread->td_locks++; 337 } 338 339 return (rval); 340 } 341 342 void 343 _sx_sunlock(struct sx *sx, const char *file, int line) 344 { 345 346 if (SCHEDULER_STOPPED()) 347 return; 348 MPASS(curthread != NULL); 349 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 350 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 351 _sx_assert(sx, SA_SLOCKED, file, line); 352 curthread->td_locks--; 353 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 354 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 355 __sx_sunlock(sx, file, line); 356 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); 357 } 358 359 void 360 _sx_xunlock(struct sx *sx, const char *file, int line) 361 { 362 363 if (SCHEDULER_STOPPED()) 364 return; 365 MPASS(curthread != NULL); 366 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 367 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 368 _sx_assert(sx, SA_XLOCKED, file, line); 369 curthread->td_locks--; 370 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 371 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 372 line); 373 if (!sx_recursed(sx)) 374 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_XUNLOCK_RELEASE, sx); 375 __sx_xunlock(sx, curthread, file, line); 376 } 377 378 /* 379 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 380 * This will only succeed if this thread holds a single shared lock. 381 * Return 1 if if the upgrade succeed, 0 otherwise. 382 */ 383 int 384 _sx_try_upgrade(struct sx *sx, const char *file, int line) 385 { 386 uintptr_t x; 387 int success; 388 389 if (SCHEDULER_STOPPED()) 390 return (1); 391 392 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 393 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 394 _sx_assert(sx, SA_SLOCKED, file, line); 395 396 /* 397 * Try to switch from one shared lock to an exclusive lock. We need 398 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 399 * we will wake up the exclusive waiters when we drop the lock. 400 */ 401 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 402 success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 403 (uintptr_t)curthread | x); 404 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 405 if (success) { 406 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 407 file, line); 408 LOCKSTAT_RECORD0(LS_SX_TRYUPGRADE_UPGRADE, sx); 409 } 410 return (success); 411 } 412 413 /* 414 * Downgrade an unrecursed exclusive lock into a single shared lock. 415 */ 416 void 417 _sx_downgrade(struct sx *sx, const char *file, int line) 418 { 419 uintptr_t x; 420 int wakeup_swapper; 421 422 if (SCHEDULER_STOPPED()) 423 return; 424 425 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 426 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 427 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 428 #ifndef INVARIANTS 429 if (sx_recursed(sx)) 430 panic("downgrade of a recursed lock"); 431 #endif 432 433 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 434 435 /* 436 * Try to switch from an exclusive lock with no shared waiters 437 * to one sharer with no shared waiters. If there are 438 * exclusive waiters, we don't need to lock the sleep queue so 439 * long as we preserve the flag. We do one quick try and if 440 * that fails we grab the sleepq lock to keep the flags from 441 * changing and do it the slow way. 442 * 443 * We have to lock the sleep queue if there are shared waiters 444 * so we can wake them up. 445 */ 446 x = sx->sx_lock; 447 if (!(x & SX_LOCK_SHARED_WAITERS) && 448 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 449 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 450 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 451 return; 452 } 453 454 /* 455 * Lock the sleep queue so we can read the waiters bits 456 * without any races and wakeup any shared waiters. 457 */ 458 sleepq_lock(&sx->lock_object); 459 460 /* 461 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 462 * shared lock. If there are any shared waiters, wake them up. 463 */ 464 wakeup_swapper = 0; 465 x = sx->sx_lock; 466 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 467 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 468 if (x & SX_LOCK_SHARED_WAITERS) 469 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 470 0, SQ_SHARED_QUEUE); 471 sleepq_release(&sx->lock_object); 472 473 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 474 LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx); 475 476 if (wakeup_swapper) 477 kick_proc0(); 478 } 479 480 /* 481 * This function represents the so-called 'hard case' for sx_xlock 482 * operation. All 'easy case' failures are redirected to this. Note 483 * that ideally this would be a static function, but it needs to be 484 * accessible from at least sx.h. 485 */ 486 int 487 _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file, 488 int line) 489 { 490 GIANT_DECLARE; 491 #ifdef ADAPTIVE_SX 492 volatile struct thread *owner; 493 u_int i, spintries = 0; 494 #endif 495 uintptr_t x; 496 #ifdef LOCK_PROFILING 497 uint64_t waittime = 0; 498 int contested = 0; 499 #endif 500 int error = 0; 501 #ifdef KDTRACE_HOOKS 502 uint64_t spin_cnt = 0; 503 uint64_t sleep_cnt = 0; 504 int64_t sleep_time = 0; 505 #endif 506 507 if (SCHEDULER_STOPPED()) 508 return (0); 509 510 /* If we already hold an exclusive lock, then recurse. */ 511 if (sx_xlocked(sx)) { 512 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, 513 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 514 sx->lock_object.lo_name, file, line)); 515 sx->sx_recurse++; 516 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 517 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 518 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 519 return (0); 520 } 521 522 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 523 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 524 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 525 526 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 527 #ifdef KDTRACE_HOOKS 528 spin_cnt++; 529 #endif 530 #ifdef HWPMC_HOOKS 531 PMC_SOFT_CALL( , , lock, failed); 532 #endif 533 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 534 &waittime); 535 #ifdef ADAPTIVE_SX 536 /* 537 * If the lock is write locked and the owner is 538 * running on another CPU, spin until the owner stops 539 * running or the state of the lock changes. 540 */ 541 x = sx->sx_lock; 542 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 543 if ((x & SX_LOCK_SHARED) == 0) { 544 x = SX_OWNER(x); 545 owner = (struct thread *)x; 546 if (TD_IS_RUNNING(owner)) { 547 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 548 CTR3(KTR_LOCK, 549 "%s: spinning on %p held by %p", 550 __func__, sx, owner); 551 GIANT_SAVE(); 552 while (SX_OWNER(sx->sx_lock) == x && 553 TD_IS_RUNNING(owner)) { 554 cpu_spinwait(); 555 #ifdef KDTRACE_HOOKS 556 spin_cnt++; 557 #endif 558 } 559 continue; 560 } 561 } else if (SX_SHARERS(x) && spintries < ASX_RETRIES) { 562 GIANT_SAVE(); 563 spintries++; 564 for (i = 0; i < ASX_LOOPS; i++) { 565 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 566 CTR4(KTR_LOCK, 567 "%s: shared spinning on %p with %u and %u", 568 __func__, sx, spintries, i); 569 x = sx->sx_lock; 570 if ((x & SX_LOCK_SHARED) == 0 || 571 SX_SHARERS(x) == 0) 572 break; 573 cpu_spinwait(); 574 #ifdef KDTRACE_HOOKS 575 spin_cnt++; 576 #endif 577 } 578 if (i != ASX_LOOPS) 579 continue; 580 } 581 } 582 #endif 583 584 sleepq_lock(&sx->lock_object); 585 x = sx->sx_lock; 586 587 /* 588 * If the lock was released while spinning on the 589 * sleep queue chain lock, try again. 590 */ 591 if (x == SX_LOCK_UNLOCKED) { 592 sleepq_release(&sx->lock_object); 593 continue; 594 } 595 596 #ifdef ADAPTIVE_SX 597 /* 598 * The current lock owner might have started executing 599 * on another CPU (or the lock could have changed 600 * owners) while we were waiting on the sleep queue 601 * chain lock. If so, drop the sleep queue lock and try 602 * again. 603 */ 604 if (!(x & SX_LOCK_SHARED) && 605 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 606 owner = (struct thread *)SX_OWNER(x); 607 if (TD_IS_RUNNING(owner)) { 608 sleepq_release(&sx->lock_object); 609 continue; 610 } 611 } 612 #endif 613 614 /* 615 * If an exclusive lock was released with both shared 616 * and exclusive waiters and a shared waiter hasn't 617 * woken up and acquired the lock yet, sx_lock will be 618 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 619 * If we see that value, try to acquire it once. Note 620 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 621 * as there are other exclusive waiters still. If we 622 * fail, restart the loop. 623 */ 624 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 625 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 626 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 627 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 628 sleepq_release(&sx->lock_object); 629 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 630 __func__, sx); 631 break; 632 } 633 sleepq_release(&sx->lock_object); 634 continue; 635 } 636 637 /* 638 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 639 * than loop back and retry. 640 */ 641 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 642 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 643 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 644 sleepq_release(&sx->lock_object); 645 continue; 646 } 647 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 648 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 649 __func__, sx); 650 } 651 652 /* 653 * Since we have been unable to acquire the exclusive 654 * lock and the exclusive waiters flag is set, we have 655 * to sleep. 656 */ 657 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 658 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 659 __func__, sx); 660 661 #ifdef KDTRACE_HOOKS 662 sleep_time -= lockstat_nsecs(); 663 #endif 664 GIANT_SAVE(); 665 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 666 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 667 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 668 if (!(opts & SX_INTERRUPTIBLE)) 669 sleepq_wait(&sx->lock_object, 0); 670 else 671 error = sleepq_wait_sig(&sx->lock_object, 0); 672 #ifdef KDTRACE_HOOKS 673 sleep_time += lockstat_nsecs(); 674 sleep_cnt++; 675 #endif 676 if (error) { 677 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 678 CTR2(KTR_LOCK, 679 "%s: interruptible sleep by %p suspended by signal", 680 __func__, sx); 681 break; 682 } 683 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 684 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 685 __func__, sx); 686 } 687 688 GIANT_RESTORE(); 689 if (!error) 690 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx, 691 contested, waittime, file, line); 692 #ifdef KDTRACE_HOOKS 693 if (sleep_time) 694 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); 695 if (spin_cnt > sleep_cnt) 696 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); 697 #endif 698 return (error); 699 } 700 701 /* 702 * This function represents the so-called 'hard case' for sx_xunlock 703 * operation. All 'easy case' failures are redirected to this. Note 704 * that ideally this would be a static function, but it needs to be 705 * accessible from at least sx.h. 706 */ 707 void 708 _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 709 { 710 uintptr_t x; 711 int queue, wakeup_swapper; 712 713 if (SCHEDULER_STOPPED()) 714 return; 715 716 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 717 718 /* If the lock is recursed, then unrecurse one level. */ 719 if (sx_xlocked(sx) && sx_recursed(sx)) { 720 if ((--sx->sx_recurse) == 0) 721 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 722 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 723 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 724 return; 725 } 726 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 727 SX_LOCK_EXCLUSIVE_WAITERS)); 728 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 729 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 730 731 sleepq_lock(&sx->lock_object); 732 x = SX_LOCK_UNLOCKED; 733 734 /* 735 * The wake up algorithm here is quite simple and probably not 736 * ideal. It gives precedence to shared waiters if they are 737 * present. For this condition, we have to preserve the 738 * state of the exclusive waiters flag. 739 * If interruptible sleeps left the shared queue empty avoid a 740 * starvation for the threads sleeping on the exclusive queue by giving 741 * them precedence and cleaning up the shared waiters bit anyway. 742 */ 743 if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 && 744 sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) { 745 queue = SQ_SHARED_QUEUE; 746 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); 747 } else 748 queue = SQ_EXCLUSIVE_QUEUE; 749 750 /* Wake up all the waiters for the specific queue. */ 751 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 752 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 753 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 754 "exclusive"); 755 atomic_store_rel_ptr(&sx->sx_lock, x); 756 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, 757 queue); 758 sleepq_release(&sx->lock_object); 759 if (wakeup_swapper) 760 kick_proc0(); 761 } 762 763 /* 764 * This function represents the so-called 'hard case' for sx_slock 765 * operation. All 'easy case' failures are redirected to this. Note 766 * that ideally this would be a static function, but it needs to be 767 * accessible from at least sx.h. 768 */ 769 int 770 _sx_slock_hard(struct sx *sx, int opts, const char *file, int line) 771 { 772 GIANT_DECLARE; 773 #ifdef ADAPTIVE_SX 774 volatile struct thread *owner; 775 #endif 776 #ifdef LOCK_PROFILING 777 uint64_t waittime = 0; 778 int contested = 0; 779 #endif 780 uintptr_t x; 781 int error = 0; 782 #ifdef KDTRACE_HOOKS 783 uint64_t spin_cnt = 0; 784 uint64_t sleep_cnt = 0; 785 int64_t sleep_time = 0; 786 #endif 787 788 if (SCHEDULER_STOPPED()) 789 return (0); 790 791 /* 792 * As with rwlocks, we don't make any attempt to try to block 793 * shared locks once there is an exclusive waiter. 794 */ 795 for (;;) { 796 #ifdef KDTRACE_HOOKS 797 spin_cnt++; 798 #endif 799 x = sx->sx_lock; 800 801 /* 802 * If no other thread has an exclusive lock then try to bump up 803 * the count of sharers. Since we have to preserve the state 804 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 805 * shared lock loop back and retry. 806 */ 807 if (x & SX_LOCK_SHARED) { 808 MPASS(!(x & SX_LOCK_SHARED_WAITERS)); 809 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, 810 x + SX_ONE_SHARER)) { 811 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 812 CTR4(KTR_LOCK, 813 "%s: %p succeed %p -> %p", __func__, 814 sx, (void *)x, 815 (void *)(x + SX_ONE_SHARER)); 816 break; 817 } 818 continue; 819 } 820 #ifdef HWPMC_HOOKS 821 PMC_SOFT_CALL( , , lock, failed); 822 #endif 823 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 824 &waittime); 825 826 #ifdef ADAPTIVE_SX 827 /* 828 * If the owner is running on another CPU, spin until 829 * the owner stops running or the state of the lock 830 * changes. 831 */ 832 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 833 x = SX_OWNER(x); 834 owner = (struct thread *)x; 835 if (TD_IS_RUNNING(owner)) { 836 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 837 CTR3(KTR_LOCK, 838 "%s: spinning on %p held by %p", 839 __func__, sx, owner); 840 GIANT_SAVE(); 841 while (SX_OWNER(sx->sx_lock) == x && 842 TD_IS_RUNNING(owner)) { 843 #ifdef KDTRACE_HOOKS 844 spin_cnt++; 845 #endif 846 cpu_spinwait(); 847 } 848 continue; 849 } 850 } 851 #endif 852 853 /* 854 * Some other thread already has an exclusive lock, so 855 * start the process of blocking. 856 */ 857 sleepq_lock(&sx->lock_object); 858 x = sx->sx_lock; 859 860 /* 861 * The lock could have been released while we spun. 862 * In this case loop back and retry. 863 */ 864 if (x & SX_LOCK_SHARED) { 865 sleepq_release(&sx->lock_object); 866 continue; 867 } 868 869 #ifdef ADAPTIVE_SX 870 /* 871 * If the owner is running on another CPU, spin until 872 * the owner stops running or the state of the lock 873 * changes. 874 */ 875 if (!(x & SX_LOCK_SHARED) && 876 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 877 owner = (struct thread *)SX_OWNER(x); 878 if (TD_IS_RUNNING(owner)) { 879 sleepq_release(&sx->lock_object); 880 continue; 881 } 882 } 883 #endif 884 885 /* 886 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 887 * fail to set it drop the sleep queue lock and loop 888 * back. 889 */ 890 if (!(x & SX_LOCK_SHARED_WAITERS)) { 891 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 892 x | SX_LOCK_SHARED_WAITERS)) { 893 sleepq_release(&sx->lock_object); 894 continue; 895 } 896 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 897 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 898 __func__, sx); 899 } 900 901 /* 902 * Since we have been unable to acquire the shared lock, 903 * we have to sleep. 904 */ 905 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 906 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 907 __func__, sx); 908 909 #ifdef KDTRACE_HOOKS 910 sleep_time -= lockstat_nsecs(); 911 #endif 912 GIANT_SAVE(); 913 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 914 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 915 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 916 if (!(opts & SX_INTERRUPTIBLE)) 917 sleepq_wait(&sx->lock_object, 0); 918 else 919 error = sleepq_wait_sig(&sx->lock_object, 0); 920 #ifdef KDTRACE_HOOKS 921 sleep_time += lockstat_nsecs(); 922 sleep_cnt++; 923 #endif 924 if (error) { 925 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 926 CTR2(KTR_LOCK, 927 "%s: interruptible sleep by %p suspended by signal", 928 __func__, sx); 929 break; 930 } 931 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 932 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 933 __func__, sx); 934 } 935 if (error == 0) 936 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 937 contested, waittime, file, line); 938 #ifdef KDTRACE_HOOKS 939 if (sleep_time) 940 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); 941 if (spin_cnt > sleep_cnt) 942 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); 943 #endif 944 GIANT_RESTORE(); 945 return (error); 946 } 947 948 /* 949 * This function represents the so-called 'hard case' for sx_sunlock 950 * operation. All 'easy case' failures are redirected to this. Note 951 * that ideally this would be a static function, but it needs to be 952 * accessible from at least sx.h. 953 */ 954 void 955 _sx_sunlock_hard(struct sx *sx, const char *file, int line) 956 { 957 uintptr_t x; 958 int wakeup_swapper; 959 960 if (SCHEDULER_STOPPED()) 961 return; 962 963 for (;;) { 964 x = sx->sx_lock; 965 966 /* 967 * We should never have sharers while at least one thread 968 * holds a shared lock. 969 */ 970 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), 971 ("%s: waiting sharers", __func__)); 972 973 /* 974 * See if there is more than one shared lock held. If 975 * so, just drop one and return. 976 */ 977 if (SX_SHARERS(x) > 1) { 978 if (atomic_cmpset_rel_ptr(&sx->sx_lock, x, 979 x - SX_ONE_SHARER)) { 980 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 981 CTR4(KTR_LOCK, 982 "%s: %p succeeded %p -> %p", 983 __func__, sx, (void *)x, 984 (void *)(x - SX_ONE_SHARER)); 985 break; 986 } 987 continue; 988 } 989 990 /* 991 * If there aren't any waiters for an exclusive lock, 992 * then try to drop it quickly. 993 */ 994 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 995 MPASS(x == SX_SHARERS_LOCK(1)); 996 if (atomic_cmpset_rel_ptr(&sx->sx_lock, 997 SX_SHARERS_LOCK(1), SX_LOCK_UNLOCKED)) { 998 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 999 CTR2(KTR_LOCK, "%s: %p last succeeded", 1000 __func__, sx); 1001 break; 1002 } 1003 continue; 1004 } 1005 1006 /* 1007 * At this point, there should just be one sharer with 1008 * exclusive waiters. 1009 */ 1010 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 1011 1012 sleepq_lock(&sx->lock_object); 1013 1014 /* 1015 * Wake up semantic here is quite simple: 1016 * Just wake up all the exclusive waiters. 1017 * Note that the state of the lock could have changed, 1018 * so if it fails loop back and retry. 1019 */ 1020 if (!atomic_cmpset_rel_ptr(&sx->sx_lock, 1021 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 1022 SX_LOCK_UNLOCKED)) { 1023 sleepq_release(&sx->lock_object); 1024 continue; 1025 } 1026 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1027 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 1028 "exclusive queue", __func__, sx); 1029 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 1030 0, SQ_EXCLUSIVE_QUEUE); 1031 sleepq_release(&sx->lock_object); 1032 if (wakeup_swapper) 1033 kick_proc0(); 1034 break; 1035 } 1036 } 1037 1038 #ifdef INVARIANT_SUPPORT 1039 #ifndef INVARIANTS 1040 #undef _sx_assert 1041 #endif 1042 1043 /* 1044 * In the non-WITNESS case, sx_assert() can only detect that at least 1045 * *some* thread owns an slock, but it cannot guarantee that *this* 1046 * thread owns an slock. 1047 */ 1048 void 1049 _sx_assert(struct sx *sx, int what, const char *file, int line) 1050 { 1051 #ifndef WITNESS 1052 int slocked = 0; 1053 #endif 1054 1055 if (panicstr != NULL) 1056 return; 1057 switch (what) { 1058 case SA_SLOCKED: 1059 case SA_SLOCKED | SA_NOTRECURSED: 1060 case SA_SLOCKED | SA_RECURSED: 1061 #ifndef WITNESS 1062 slocked = 1; 1063 /* FALLTHROUGH */ 1064 #endif 1065 case SA_LOCKED: 1066 case SA_LOCKED | SA_NOTRECURSED: 1067 case SA_LOCKED | SA_RECURSED: 1068 #ifdef WITNESS 1069 witness_assert(&sx->lock_object, what, file, line); 1070 #else 1071 /* 1072 * If some other thread has an exclusive lock or we 1073 * have one and are asserting a shared lock, fail. 1074 * Also, if no one has a lock at all, fail. 1075 */ 1076 if (sx->sx_lock == SX_LOCK_UNLOCKED || 1077 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 1078 sx_xholder(sx) != curthread))) 1079 panic("Lock %s not %slocked @ %s:%d\n", 1080 sx->lock_object.lo_name, slocked ? "share " : "", 1081 file, line); 1082 1083 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 1084 if (sx_recursed(sx)) { 1085 if (what & SA_NOTRECURSED) 1086 panic("Lock %s recursed @ %s:%d\n", 1087 sx->lock_object.lo_name, file, 1088 line); 1089 } else if (what & SA_RECURSED) 1090 panic("Lock %s not recursed @ %s:%d\n", 1091 sx->lock_object.lo_name, file, line); 1092 } 1093 #endif 1094 break; 1095 case SA_XLOCKED: 1096 case SA_XLOCKED | SA_NOTRECURSED: 1097 case SA_XLOCKED | SA_RECURSED: 1098 if (sx_xholder(sx) != curthread) 1099 panic("Lock %s not exclusively locked @ %s:%d\n", 1100 sx->lock_object.lo_name, file, line); 1101 if (sx_recursed(sx)) { 1102 if (what & SA_NOTRECURSED) 1103 panic("Lock %s recursed @ %s:%d\n", 1104 sx->lock_object.lo_name, file, line); 1105 } else if (what & SA_RECURSED) 1106 panic("Lock %s not recursed @ %s:%d\n", 1107 sx->lock_object.lo_name, file, line); 1108 break; 1109 case SA_UNLOCKED: 1110 #ifdef WITNESS 1111 witness_assert(&sx->lock_object, what, file, line); 1112 #else 1113 /* 1114 * If we hold an exclusve lock fail. We can't 1115 * reliably check to see if we hold a shared lock or 1116 * not. 1117 */ 1118 if (sx_xholder(sx) == curthread) 1119 panic("Lock %s exclusively locked @ %s:%d\n", 1120 sx->lock_object.lo_name, file, line); 1121 #endif 1122 break; 1123 default: 1124 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 1125 line); 1126 } 1127 } 1128 #endif /* INVARIANT_SUPPORT */ 1129 1130 #ifdef DDB 1131 static void 1132 db_show_sx(struct lock_object *lock) 1133 { 1134 struct thread *td; 1135 struct sx *sx; 1136 1137 sx = (struct sx *)lock; 1138 1139 db_printf(" state: "); 1140 if (sx->sx_lock == SX_LOCK_UNLOCKED) 1141 db_printf("UNLOCKED\n"); 1142 else if (sx->sx_lock == SX_LOCK_DESTROYED) { 1143 db_printf("DESTROYED\n"); 1144 return; 1145 } else if (sx->sx_lock & SX_LOCK_SHARED) 1146 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 1147 else { 1148 td = sx_xholder(sx); 1149 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1150 td->td_tid, td->td_proc->p_pid, td->td_name); 1151 if (sx_recursed(sx)) 1152 db_printf(" recursed: %d\n", sx->sx_recurse); 1153 } 1154 1155 db_printf(" waiters: "); 1156 switch(sx->sx_lock & 1157 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 1158 case SX_LOCK_SHARED_WAITERS: 1159 db_printf("shared\n"); 1160 break; 1161 case SX_LOCK_EXCLUSIVE_WAITERS: 1162 db_printf("exclusive\n"); 1163 break; 1164 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 1165 db_printf("exclusive and shared\n"); 1166 break; 1167 default: 1168 db_printf("none\n"); 1169 } 1170 } 1171 1172 /* 1173 * Check to see if a thread that is blocked on a sleep queue is actually 1174 * blocked on an sx lock. If so, output some details and return true. 1175 * If the lock has an exclusive owner, return that in *ownerp. 1176 */ 1177 int 1178 sx_chain(struct thread *td, struct thread **ownerp) 1179 { 1180 struct sx *sx; 1181 1182 /* 1183 * Check to see if this thread is blocked on an sx lock. 1184 * First, we check the lock class. If that is ok, then we 1185 * compare the lock name against the wait message. 1186 */ 1187 sx = td->td_wchan; 1188 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 1189 sx->lock_object.lo_name != td->td_wmesg) 1190 return (0); 1191 1192 /* We think we have an sx lock, so output some details. */ 1193 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 1194 *ownerp = sx_xholder(sx); 1195 if (sx->sx_lock & SX_LOCK_SHARED) 1196 db_printf("SLOCK (count %ju)\n", 1197 (uintmax_t)SX_SHARERS(sx->sx_lock)); 1198 else 1199 db_printf("XLOCK\n"); 1200 return (1); 1201 } 1202 #endif
Cache object: a7bb538238751d8f9ca0ab9ea59e555d
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]