Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_umtx.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice unmodified, this list of conditions, and the following 11 * disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_compat.h" 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/limits.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/priv.h> 39 #include <sys/proc.h> 40 #include <sys/sched.h> 41 #include <sys/smp.h> 42 #include <sys/sysctl.h> 43 #include <sys/sysent.h> 44 #include <sys/systm.h> 45 #include <sys/sysproto.h> 46 #include <sys/eventhandler.h> 47 #include <sys/umtx.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/pmap.h> 52 #include <vm/vm_map.h> 53 #include <vm/vm_object.h> 54 55 #include <machine/cpu.h> 56 57 #ifdef COMPAT_IA32 58 #include <compat/freebsd32/freebsd32_proto.h> 59 #endif 60 61 #define TYPE_SIMPLE_LOCK 0 62 #define TYPE_SIMPLE_WAIT 1 63 #define TYPE_NORMAL_UMUTEX 2 64 #define TYPE_PI_UMUTEX 3 65 #define TYPE_PP_UMUTEX 4 66 #define TYPE_CV 5 67 68 /* Key to represent a unique userland synchronous object */ 69 struct umtx_key { 70 int hash; 71 int type; 72 int shared; 73 union { 74 struct { 75 vm_object_t object; 76 uintptr_t offset; 77 } shared; 78 struct { 79 struct vmspace *vs; 80 uintptr_t addr; 81 } private; 82 struct { 83 void *a; 84 uintptr_t b; 85 } both; 86 } info; 87 }; 88 89 /* Priority inheritance mutex info. */ 90 struct umtx_pi { 91 /* Owner thread */ 92 struct thread *pi_owner; 93 94 /* Reference count */ 95 int pi_refcount; 96 97 /* List entry to link umtx holding by thread */ 98 TAILQ_ENTRY(umtx_pi) pi_link; 99 100 /* List entry in hash */ 101 TAILQ_ENTRY(umtx_pi) pi_hashlink; 102 103 /* List for waiters */ 104 TAILQ_HEAD(,umtx_q) pi_blocked; 105 106 /* Identify a userland lock object */ 107 struct umtx_key pi_key; 108 }; 109 110 /* A userland synchronous object user. */ 111 struct umtx_q { 112 /* Linked list for the hash. */ 113 TAILQ_ENTRY(umtx_q) uq_link; 114 115 /* Umtx key. */ 116 struct umtx_key uq_key; 117 118 /* Umtx flags. */ 119 int uq_flags; 120 #define UQF_UMTXQ 0x0001 121 122 /* The thread waits on. */ 123 struct thread *uq_thread; 124 125 /* 126 * Blocked on PI mutex. read can use chain lock 127 * or umtx_lock, write must have both chain lock and 128 * umtx_lock being hold. 129 */ 130 struct umtx_pi *uq_pi_blocked; 131 132 /* On blocked list */ 133 TAILQ_ENTRY(umtx_q) uq_lockq; 134 135 /* Thread contending with us */ 136 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 137 138 /* Inherited priority from PP mutex */ 139 u_char uq_inherited_pri; 140 }; 141 142 TAILQ_HEAD(umtxq_head, umtx_q); 143 144 /* Userland lock object's wait-queue chain */ 145 struct umtxq_chain { 146 /* Lock for this chain. */ 147 struct mtx uc_lock; 148 149 /* List of sleep queues. */ 150 struct umtxq_head uc_queue; 151 152 /* Busy flag */ 153 char uc_busy; 154 155 /* Chain lock waiters */ 156 int uc_waiters; 157 158 /* All PI in the list */ 159 TAILQ_HEAD(,umtx_pi) uc_pi_list; 160 }; 161 162 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) 163 164 /* 165 * Don't propagate time-sharing priority, there is a security reason, 166 * a user can simply introduce PI-mutex, let thread A lock the mutex, 167 * and let another thread B block on the mutex, because B is 168 * sleeping, its priority will be boosted, this causes A's priority to 169 * be boosted via priority propagating too and will never be lowered even 170 * if it is using 100%CPU, this is unfair to other processes. 171 */ 172 173 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ 174 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ 175 PRI_MAX_TIMESHARE : (td)->td_user_pri) 176 177 #define GOLDEN_RATIO_PRIME 2654404609U 178 #define UMTX_CHAINS 128 179 #define UMTX_SHIFTS (__WORD_BIT - 7) 180 181 #define THREAD_SHARE 0 182 #define PROCESS_SHARE 1 183 #define AUTO_SHARE 2 184 185 #define GET_SHARE(flags) \ 186 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) 187 188 static uma_zone_t umtx_pi_zone; 189 static struct umtxq_chain umtxq_chains[UMTX_CHAINS]; 190 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 191 static int umtx_pi_allocated; 192 193 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); 194 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, 195 &umtx_pi_allocated, 0, "Allocated umtx_pi"); 196 197 static void umtxq_sysinit(void *); 198 static void umtxq_hash(struct umtx_key *key); 199 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); 200 static void umtxq_lock(struct umtx_key *key); 201 static void umtxq_unlock(struct umtx_key *key); 202 static void umtxq_busy(struct umtx_key *key); 203 static void umtxq_unbusy(struct umtx_key *key); 204 static void umtxq_insert(struct umtx_q *uq); 205 static void umtxq_remove(struct umtx_q *uq); 206 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo); 207 static int umtxq_count(struct umtx_key *key); 208 static int umtxq_signal(struct umtx_key *key, int nr_wakeup); 209 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); 210 static int umtx_key_get(void *addr, int type, int share, 211 struct umtx_key *key); 212 static void umtx_key_release(struct umtx_key *key); 213 static struct umtx_pi *umtx_pi_alloc(int); 214 static void umtx_pi_free(struct umtx_pi *pi); 215 static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri); 216 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); 217 static void umtx_thread_cleanup(struct thread *td); 218 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, 219 struct image_params *imgp __unused); 220 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); 221 222 static struct mtx umtx_lock; 223 224 static void 225 umtxq_sysinit(void *arg __unused) 226 { 227 int i; 228 229 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), 230 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 231 for (i = 0; i < UMTX_CHAINS; ++i) { 232 mtx_init(&umtxq_chains[i].uc_lock, "umtxql", NULL, 233 MTX_DEF | MTX_DUPOK); 234 TAILQ_INIT(&umtxq_chains[i].uc_queue); 235 TAILQ_INIT(&umtxq_chains[i].uc_pi_list); 236 umtxq_chains[i].uc_busy = 0; 237 umtxq_chains[i].uc_waiters = 0; 238 } 239 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); 240 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, 241 EVENTHANDLER_PRI_ANY); 242 } 243 244 struct umtx_q * 245 umtxq_alloc(void) 246 { 247 struct umtx_q *uq; 248 249 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); 250 TAILQ_INIT(&uq->uq_pi_contested); 251 uq->uq_inherited_pri = PRI_MAX; 252 return (uq); 253 } 254 255 void 256 umtxq_free(struct umtx_q *uq) 257 { 258 free(uq, M_UMTX); 259 } 260 261 static inline void 262 umtxq_hash(struct umtx_key *key) 263 { 264 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; 265 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; 266 } 267 268 static inline int 269 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 270 { 271 return (k1->type == k2->type && 272 k1->info.both.a == k2->info.both.a && 273 k1->info.both.b == k2->info.both.b); 274 } 275 276 static inline struct umtxq_chain * 277 umtxq_getchain(struct umtx_key *key) 278 { 279 return (&umtxq_chains[key->hash]); 280 } 281 282 /* 283 * Set chain to busy state when following operation 284 * may be blocked (kernel mutex can not be used). 285 */ 286 static inline void 287 umtxq_busy(struct umtx_key *key) 288 { 289 struct umtxq_chain *uc; 290 291 uc = umtxq_getchain(key); 292 mtx_assert(&uc->uc_lock, MA_OWNED); 293 while (uc->uc_busy != 0) { 294 uc->uc_waiters++; 295 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); 296 uc->uc_waiters--; 297 } 298 uc->uc_busy = 1; 299 } 300 301 /* 302 * Unbusy a chain. 303 */ 304 static inline void 305 umtxq_unbusy(struct umtx_key *key) 306 { 307 struct umtxq_chain *uc; 308 309 uc = umtxq_getchain(key); 310 mtx_assert(&uc->uc_lock, MA_OWNED); 311 KASSERT(uc->uc_busy != 0, ("not busy")); 312 uc->uc_busy = 0; 313 if (uc->uc_waiters) 314 wakeup_one(uc); 315 } 316 317 /* 318 * Lock a chain. 319 */ 320 static inline void 321 umtxq_lock(struct umtx_key *key) 322 { 323 struct umtxq_chain *uc; 324 325 uc = umtxq_getchain(key); 326 mtx_lock(&uc->uc_lock); 327 } 328 329 /* 330 * Unlock a chain. 331 */ 332 static inline void 333 umtxq_unlock(struct umtx_key *key) 334 { 335 struct umtxq_chain *uc; 336 337 uc = umtxq_getchain(key); 338 mtx_unlock(&uc->uc_lock); 339 } 340 341 /* 342 * Insert a thread onto the umtx queue. 343 */ 344 static inline void 345 umtxq_insert(struct umtx_q *uq) 346 { 347 struct umtxq_chain *uc; 348 349 uc = umtxq_getchain(&uq->uq_key); 350 UMTXQ_LOCKED_ASSERT(uc); 351 TAILQ_INSERT_TAIL(&uc->uc_queue, uq, uq_link); 352 uq->uq_flags |= UQF_UMTXQ; 353 } 354 355 /* 356 * Remove thread from the umtx queue. 357 */ 358 static inline void 359 umtxq_remove(struct umtx_q *uq) 360 { 361 struct umtxq_chain *uc; 362 363 uc = umtxq_getchain(&uq->uq_key); 364 UMTXQ_LOCKED_ASSERT(uc); 365 if (uq->uq_flags & UQF_UMTXQ) { 366 TAILQ_REMOVE(&uc->uc_queue, uq, uq_link); 367 uq->uq_flags &= ~UQF_UMTXQ; 368 } 369 } 370 371 /* 372 * Check if there are multiple waiters 373 */ 374 static int 375 umtxq_count(struct umtx_key *key) 376 { 377 struct umtxq_chain *uc; 378 struct umtx_q *uq; 379 int count = 0; 380 381 uc = umtxq_getchain(key); 382 UMTXQ_LOCKED_ASSERT(uc); 383 TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) { 384 if (umtx_key_match(&uq->uq_key, key)) { 385 if (++count > 1) 386 break; 387 } 388 } 389 return (count); 390 } 391 392 /* 393 * Check if there are multiple PI waiters and returns first 394 * waiter. 395 */ 396 static int 397 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) 398 { 399 struct umtxq_chain *uc; 400 struct umtx_q *uq; 401 int count = 0; 402 403 *first = NULL; 404 uc = umtxq_getchain(key); 405 UMTXQ_LOCKED_ASSERT(uc); 406 TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) { 407 if (umtx_key_match(&uq->uq_key, key)) { 408 if (++count > 1) 409 break; 410 *first = uq; 411 } 412 } 413 return (count); 414 } 415 416 /* 417 * Wake up threads waiting on an userland object. 418 */ 419 static int 420 umtxq_signal(struct umtx_key *key, int n_wake) 421 { 422 struct umtxq_chain *uc; 423 struct umtx_q *uq, *next; 424 int ret; 425 426 ret = 0; 427 uc = umtxq_getchain(key); 428 UMTXQ_LOCKED_ASSERT(uc); 429 TAILQ_FOREACH_SAFE(uq, &uc->uc_queue, uq_link, next) { 430 if (umtx_key_match(&uq->uq_key, key)) { 431 umtxq_remove(uq); 432 wakeup(uq); 433 if (++ret >= n_wake) 434 break; 435 } 436 } 437 return (ret); 438 } 439 440 /* 441 * Wake up specified thread. 442 */ 443 static inline void 444 umtxq_signal_thread(struct umtx_q *uq) 445 { 446 struct umtxq_chain *uc; 447 448 uc = umtxq_getchain(&uq->uq_key); 449 UMTXQ_LOCKED_ASSERT(uc); 450 umtxq_remove(uq); 451 wakeup(uq); 452 } 453 454 /* 455 * Put thread into sleep state, before sleeping, check if 456 * thread was removed from umtx queue. 457 */ 458 static inline int 459 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo) 460 { 461 struct umtxq_chain *uc; 462 int error; 463 464 uc = umtxq_getchain(&uq->uq_key); 465 UMTXQ_LOCKED_ASSERT(uc); 466 if (!(uq->uq_flags & UQF_UMTXQ)) 467 return (0); 468 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 469 if (error == EWOULDBLOCK) 470 error = ETIMEDOUT; 471 return (error); 472 } 473 474 /* 475 * Convert userspace address into unique logical address. 476 */ 477 static int 478 umtx_key_get(void *addr, int type, int share, struct umtx_key *key) 479 { 480 struct thread *td = curthread; 481 vm_map_t map; 482 vm_map_entry_t entry; 483 vm_pindex_t pindex; 484 vm_prot_t prot; 485 boolean_t wired; 486 487 key->type = type; 488 if (share == THREAD_SHARE) { 489 key->shared = 0; 490 key->info.private.vs = td->td_proc->p_vmspace; 491 key->info.private.addr = (uintptr_t)addr; 492 } else { 493 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); 494 map = &td->td_proc->p_vmspace->vm_map; 495 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, 496 &entry, &key->info.shared.object, &pindex, &prot, 497 &wired) != KERN_SUCCESS) { 498 return EFAULT; 499 } 500 501 if ((share == PROCESS_SHARE) || 502 (share == AUTO_SHARE && 503 VM_INHERIT_SHARE == entry->inheritance)) { 504 key->shared = 1; 505 key->info.shared.offset = entry->offset + entry->start - 506 (vm_offset_t)addr; 507 vm_object_reference(key->info.shared.object); 508 } else { 509 key->shared = 0; 510 key->info.private.vs = td->td_proc->p_vmspace; 511 key->info.private.addr = (uintptr_t)addr; 512 } 513 vm_map_lookup_done(map, entry); 514 } 515 516 umtxq_hash(key); 517 return (0); 518 } 519 520 /* 521 * Release key. 522 */ 523 static inline void 524 umtx_key_release(struct umtx_key *key) 525 { 526 if (key->shared) 527 vm_object_deallocate(key->info.shared.object); 528 } 529 530 /* 531 * Lock a umtx object. 532 */ 533 static int 534 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo) 535 { 536 struct umtx_q *uq; 537 u_long owner; 538 u_long old; 539 int error = 0; 540 541 uq = td->td_umtxq; 542 543 /* 544 * Care must be exercised when dealing with umtx structure. It 545 * can fault on any access. 546 */ 547 for (;;) { 548 /* 549 * Try the uncontested case. This should be done in userland. 550 */ 551 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id); 552 553 /* The acquire succeeded. */ 554 if (owner == UMTX_UNOWNED) 555 return (0); 556 557 /* The address was invalid. */ 558 if (owner == -1) 559 return (EFAULT); 560 561 /* If no one owns it but it is contested try to acquire it. */ 562 if (owner == UMTX_CONTESTED) { 563 owner = casuword(&umtx->u_owner, 564 UMTX_CONTESTED, id | UMTX_CONTESTED); 565 566 if (owner == UMTX_CONTESTED) 567 return (0); 568 569 /* The address was invalid. */ 570 if (owner == -1) 571 return (EFAULT); 572 573 /* If this failed the lock has changed, restart. */ 574 continue; 575 } 576 577 /* 578 * If we caught a signal, we have retried and now 579 * exit immediately. 580 */ 581 if (error != 0) 582 return (error); 583 584 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, 585 AUTO_SHARE, &uq->uq_key)) != 0) 586 return (error); 587 588 umtxq_lock(&uq->uq_key); 589 umtxq_busy(&uq->uq_key); 590 umtxq_insert(uq); 591 umtxq_unbusy(&uq->uq_key); 592 umtxq_unlock(&uq->uq_key); 593 594 /* 595 * Set the contested bit so that a release in user space 596 * knows to use the system call for unlock. If this fails 597 * either some one else has acquired the lock or it has been 598 * released. 599 */ 600 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED); 601 602 /* The address was invalid. */ 603 if (old == -1) { 604 umtxq_lock(&uq->uq_key); 605 umtxq_remove(uq); 606 umtxq_unlock(&uq->uq_key); 607 umtx_key_release(&uq->uq_key); 608 return (EFAULT); 609 } 610 611 /* 612 * We set the contested bit, sleep. Otherwise the lock changed 613 * and we need to retry or we lost a race to the thread 614 * unlocking the umtx. 615 */ 616 umtxq_lock(&uq->uq_key); 617 if (old == owner) 618 error = umtxq_sleep(uq, "umtx", timo); 619 umtxq_remove(uq); 620 umtxq_unlock(&uq->uq_key); 621 umtx_key_release(&uq->uq_key); 622 } 623 624 return (0); 625 } 626 627 /* 628 * Lock a umtx object. 629 */ 630 static int 631 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, 632 struct timespec *timeout) 633 { 634 struct timespec ts, ts2, ts3; 635 struct timeval tv; 636 int error; 637 638 if (timeout == NULL) { 639 error = _do_lock_umtx(td, umtx, id, 0); 640 /* Mutex locking is restarted if it is interrupted. */ 641 if (error == EINTR) 642 error = ERESTART; 643 } else { 644 getnanouptime(&ts); 645 timespecadd(&ts, timeout); 646 TIMESPEC_TO_TIMEVAL(&tv, timeout); 647 for (;;) { 648 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv)); 649 if (error != ETIMEDOUT) 650 break; 651 getnanouptime(&ts2); 652 if (timespeccmp(&ts2, &ts, >=)) { 653 error = ETIMEDOUT; 654 break; 655 } 656 ts3 = ts; 657 timespecsub(&ts3, &ts2); 658 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 659 } 660 /* Timed-locking is not restarted. */ 661 if (error == ERESTART) 662 error = EINTR; 663 } 664 return (error); 665 } 666 667 /* 668 * Unlock a umtx object. 669 */ 670 static int 671 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id) 672 { 673 struct umtx_key key; 674 u_long owner; 675 u_long old; 676 int error; 677 int count; 678 679 /* 680 * Make sure we own this mtx. 681 */ 682 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner)); 683 if (owner == -1) 684 return (EFAULT); 685 686 if ((owner & ~UMTX_CONTESTED) != id) 687 return (EPERM); 688 689 /* This should be done in userland */ 690 if ((owner & UMTX_CONTESTED) == 0) { 691 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED); 692 if (old == -1) 693 return (EFAULT); 694 if (old == owner) 695 return (0); 696 owner = old; 697 } 698 699 /* We should only ever be in here for contested locks */ 700 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, 701 &key)) != 0) 702 return (error); 703 704 umtxq_lock(&key); 705 umtxq_busy(&key); 706 count = umtxq_count(&key); 707 umtxq_unlock(&key); 708 709 /* 710 * When unlocking the umtx, it must be marked as unowned if 711 * there is zero or one thread only waiting for it. 712 * Otherwise, it must be marked as contested. 713 */ 714 old = casuword(&umtx->u_owner, owner, 715 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); 716 umtxq_lock(&key); 717 umtxq_signal(&key,1); 718 umtxq_unbusy(&key); 719 umtxq_unlock(&key); 720 umtx_key_release(&key); 721 if (old == -1) 722 return (EFAULT); 723 if (old != owner) 724 return (EINVAL); 725 return (0); 726 } 727 728 #ifdef COMPAT_IA32 729 730 /* 731 * Lock a umtx object. 732 */ 733 static int 734 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo) 735 { 736 struct umtx_q *uq; 737 uint32_t owner; 738 uint32_t old; 739 int error = 0; 740 741 uq = td->td_umtxq; 742 743 /* 744 * Care must be exercised when dealing with umtx structure. It 745 * can fault on any access. 746 */ 747 for (;;) { 748 /* 749 * Try the uncontested case. This should be done in userland. 750 */ 751 owner = casuword32(m, UMUTEX_UNOWNED, id); 752 753 /* The acquire succeeded. */ 754 if (owner == UMUTEX_UNOWNED) 755 return (0); 756 757 /* The address was invalid. */ 758 if (owner == -1) 759 return (EFAULT); 760 761 /* If no one owns it but it is contested try to acquire it. */ 762 if (owner == UMUTEX_CONTESTED) { 763 owner = casuword32(m, 764 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 765 if (owner == UMUTEX_CONTESTED) 766 return (0); 767 768 /* The address was invalid. */ 769 if (owner == -1) 770 return (EFAULT); 771 772 /* If this failed the lock has changed, restart. */ 773 continue; 774 } 775 776 /* 777 * If we caught a signal, we have retried and now 778 * exit immediately. 779 */ 780 if (error != 0) 781 return (error); 782 783 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, 784 AUTO_SHARE, &uq->uq_key)) != 0) 785 return (error); 786 787 umtxq_lock(&uq->uq_key); 788 umtxq_busy(&uq->uq_key); 789 umtxq_insert(uq); 790 umtxq_unbusy(&uq->uq_key); 791 umtxq_unlock(&uq->uq_key); 792 793 /* 794 * Set the contested bit so that a release in user space 795 * knows to use the system call for unlock. If this fails 796 * either some one else has acquired the lock or it has been 797 * released. 798 */ 799 old = casuword32(m, owner, owner | UMUTEX_CONTESTED); 800 801 /* The address was invalid. */ 802 if (old == -1) { 803 umtxq_lock(&uq->uq_key); 804 umtxq_remove(uq); 805 umtxq_unlock(&uq->uq_key); 806 umtx_key_release(&uq->uq_key); 807 return (EFAULT); 808 } 809 810 /* 811 * We set the contested bit, sleep. Otherwise the lock changed 812 * and we need to retry or we lost a race to the thread 813 * unlocking the umtx. 814 */ 815 umtxq_lock(&uq->uq_key); 816 if (old == owner) 817 error = umtxq_sleep(uq, "umtx", timo); 818 umtxq_remove(uq); 819 umtxq_unlock(&uq->uq_key); 820 umtx_key_release(&uq->uq_key); 821 } 822 823 return (0); 824 } 825 826 /* 827 * Lock a umtx object. 828 */ 829 static int 830 do_lock_umtx32(struct thread *td, void *m, uint32_t id, 831 struct timespec *timeout) 832 { 833 struct timespec ts, ts2, ts3; 834 struct timeval tv; 835 int error; 836 837 if (timeout == NULL) { 838 error = _do_lock_umtx32(td, m, id, 0); 839 /* Mutex locking is restarted if it is interrupted. */ 840 if (error == EINTR) 841 error = ERESTART; 842 } else { 843 getnanouptime(&ts); 844 timespecadd(&ts, timeout); 845 TIMESPEC_TO_TIMEVAL(&tv, timeout); 846 for (;;) { 847 error = _do_lock_umtx32(td, m, id, tvtohz(&tv)); 848 if (error != ETIMEDOUT) 849 break; 850 getnanouptime(&ts2); 851 if (timespeccmp(&ts2, &ts, >=)) { 852 error = ETIMEDOUT; 853 break; 854 } 855 ts3 = ts; 856 timespecsub(&ts3, &ts2); 857 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 858 } 859 /* Timed-locking is not restarted. */ 860 if (error == ERESTART) 861 error = EINTR; 862 } 863 return (error); 864 } 865 866 /* 867 * Unlock a umtx object. 868 */ 869 static int 870 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id) 871 { 872 struct umtx_key key; 873 uint32_t owner; 874 uint32_t old; 875 int error; 876 int count; 877 878 /* 879 * Make sure we own this mtx. 880 */ 881 owner = fuword32(m); 882 if (owner == -1) 883 return (EFAULT); 884 885 if ((owner & ~UMUTEX_CONTESTED) != id) 886 return (EPERM); 887 888 /* This should be done in userland */ 889 if ((owner & UMUTEX_CONTESTED) == 0) { 890 old = casuword32(m, owner, UMUTEX_UNOWNED); 891 if (old == -1) 892 return (EFAULT); 893 if (old == owner) 894 return (0); 895 owner = old; 896 } 897 898 /* We should only ever be in here for contested locks */ 899 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE, 900 &key)) != 0) 901 return (error); 902 903 umtxq_lock(&key); 904 umtxq_busy(&key); 905 count = umtxq_count(&key); 906 umtxq_unlock(&key); 907 908 /* 909 * When unlocking the umtx, it must be marked as unowned if 910 * there is zero or one thread only waiting for it. 911 * Otherwise, it must be marked as contested. 912 */ 913 old = casuword32(m, owner, 914 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 915 umtxq_lock(&key); 916 umtxq_signal(&key,1); 917 umtxq_unbusy(&key); 918 umtxq_unlock(&key); 919 umtx_key_release(&key); 920 if (old == -1) 921 return (EFAULT); 922 if (old != owner) 923 return (EINVAL); 924 return (0); 925 } 926 #endif 927 928 /* 929 * Fetch and compare value, sleep on the address if value is not changed. 930 */ 931 static int 932 do_wait(struct thread *td, void *addr, u_long id, 933 struct timespec *timeout, int compat32) 934 { 935 struct umtx_q *uq; 936 struct timespec ts, ts2, ts3; 937 struct timeval tv; 938 u_long tmp; 939 int error = 0; 940 941 uq = td->td_umtxq; 942 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, AUTO_SHARE, 943 &uq->uq_key)) != 0) 944 return (error); 945 946 umtxq_lock(&uq->uq_key); 947 umtxq_insert(uq); 948 umtxq_unlock(&uq->uq_key); 949 if (compat32 == 0) 950 tmp = fuword(addr); 951 else 952 tmp = fuword32(addr); 953 if (tmp != id) { 954 umtxq_lock(&uq->uq_key); 955 umtxq_remove(uq); 956 umtxq_unlock(&uq->uq_key); 957 } else if (timeout == NULL) { 958 umtxq_lock(&uq->uq_key); 959 error = umtxq_sleep(uq, "uwait", 0); 960 umtxq_remove(uq); 961 umtxq_unlock(&uq->uq_key); 962 } else { 963 getnanouptime(&ts); 964 timespecadd(&ts, timeout); 965 TIMESPEC_TO_TIMEVAL(&tv, timeout); 966 umtxq_lock(&uq->uq_key); 967 for (;;) { 968 error = umtxq_sleep(uq, "uwait", tvtohz(&tv)); 969 if (!(uq->uq_flags & UQF_UMTXQ)) 970 break; 971 if (error != ETIMEDOUT) 972 break; 973 umtxq_unlock(&uq->uq_key); 974 getnanouptime(&ts2); 975 if (timespeccmp(&ts2, &ts, >=)) { 976 error = ETIMEDOUT; 977 umtxq_lock(&uq->uq_key); 978 break; 979 } 980 ts3 = ts; 981 timespecsub(&ts3, &ts2); 982 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 983 umtxq_lock(&uq->uq_key); 984 } 985 umtxq_remove(uq); 986 umtxq_unlock(&uq->uq_key); 987 } 988 umtx_key_release(&uq->uq_key); 989 if (error == ERESTART) 990 error = EINTR; 991 return (error); 992 } 993 994 /* 995 * Wake up threads sleeping on the specified address. 996 */ 997 int 998 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake) 999 { 1000 struct umtx_key key; 1001 int ret; 1002 1003 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, AUTO_SHARE, 1004 &key)) != 0) 1005 return (ret); 1006 umtxq_lock(&key); 1007 ret = umtxq_signal(&key, n_wake); 1008 umtxq_unlock(&key); 1009 umtx_key_release(&key); 1010 return (0); 1011 } 1012 1013 /* 1014 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1015 */ 1016 static int 1017 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1018 int try) 1019 { 1020 struct umtx_q *uq; 1021 uint32_t owner, old, id; 1022 int error = 0; 1023 1024 id = td->td_tid; 1025 uq = td->td_umtxq; 1026 1027 /* 1028 * Care must be exercised when dealing with umtx structure. It 1029 * can fault on any access. 1030 */ 1031 for (;;) { 1032 /* 1033 * Try the uncontested case. This should be done in userland. 1034 */ 1035 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1036 1037 /* The acquire succeeded. */ 1038 if (owner == UMUTEX_UNOWNED) 1039 return (0); 1040 1041 /* The address was invalid. */ 1042 if (owner == -1) 1043 return (EFAULT); 1044 1045 /* If no one owns it but it is contested try to acquire it. */ 1046 if (owner == UMUTEX_CONTESTED) { 1047 owner = casuword32(&m->m_owner, 1048 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1049 1050 if (owner == UMUTEX_CONTESTED) 1051 return (0); 1052 1053 /* The address was invalid. */ 1054 if (owner == -1) 1055 return (EFAULT); 1056 1057 /* If this failed the lock has changed, restart. */ 1058 continue; 1059 } 1060 1061 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1062 (owner & ~UMUTEX_CONTESTED) == id) 1063 return (EDEADLK); 1064 1065 if (try != 0) 1066 return (EBUSY); 1067 1068 /* 1069 * If we caught a signal, we have retried and now 1070 * exit immediately. 1071 */ 1072 if (error != 0) 1073 return (error); 1074 1075 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, 1076 GET_SHARE(flags), &uq->uq_key)) != 0) 1077 return (error); 1078 1079 umtxq_lock(&uq->uq_key); 1080 umtxq_busy(&uq->uq_key); 1081 umtxq_insert(uq); 1082 umtxq_unbusy(&uq->uq_key); 1083 umtxq_unlock(&uq->uq_key); 1084 1085 /* 1086 * Set the contested bit so that a release in user space 1087 * knows to use the system call for unlock. If this fails 1088 * either some one else has acquired the lock or it has been 1089 * released. 1090 */ 1091 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1092 1093 /* The address was invalid. */ 1094 if (old == -1) { 1095 umtxq_lock(&uq->uq_key); 1096 umtxq_remove(uq); 1097 umtxq_unlock(&uq->uq_key); 1098 umtx_key_release(&uq->uq_key); 1099 return (EFAULT); 1100 } 1101 1102 /* 1103 * We set the contested bit, sleep. Otherwise the lock changed 1104 * and we need to retry or we lost a race to the thread 1105 * unlocking the umtx. 1106 */ 1107 umtxq_lock(&uq->uq_key); 1108 if (old == owner) 1109 error = umtxq_sleep(uq, "umtxn", timo); 1110 umtxq_remove(uq); 1111 umtxq_unlock(&uq->uq_key); 1112 umtx_key_release(&uq->uq_key); 1113 } 1114 1115 return (0); 1116 } 1117 1118 /* 1119 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1120 */ 1121 /* 1122 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. 1123 */ 1124 static int 1125 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) 1126 { 1127 struct umtx_key key; 1128 uint32_t owner, old, id; 1129 int error; 1130 int count; 1131 1132 id = td->td_tid; 1133 /* 1134 * Make sure we own this mtx. 1135 */ 1136 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1137 if (owner == -1) 1138 return (EFAULT); 1139 1140 if ((owner & ~UMUTEX_CONTESTED) != id) 1141 return (EPERM); 1142 1143 /* This should be done in userland */ 1144 if ((owner & UMUTEX_CONTESTED) == 0) { 1145 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1146 if (old == -1) 1147 return (EFAULT); 1148 if (old == owner) 1149 return (0); 1150 owner = old; 1151 } 1152 1153 /* We should only ever be in here for contested locks */ 1154 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1155 &key)) != 0) 1156 return (error); 1157 1158 umtxq_lock(&key); 1159 umtxq_busy(&key); 1160 count = umtxq_count(&key); 1161 umtxq_unlock(&key); 1162 1163 /* 1164 * When unlocking the umtx, it must be marked as unowned if 1165 * there is zero or one thread only waiting for it. 1166 * Otherwise, it must be marked as contested. 1167 */ 1168 old = casuword32(&m->m_owner, owner, 1169 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1170 umtxq_lock(&key); 1171 umtxq_signal(&key,1); 1172 umtxq_unbusy(&key); 1173 umtxq_unlock(&key); 1174 umtx_key_release(&key); 1175 if (old == -1) 1176 return (EFAULT); 1177 if (old != owner) 1178 return (EINVAL); 1179 return (0); 1180 } 1181 1182 static inline struct umtx_pi * 1183 umtx_pi_alloc(int flags) 1184 { 1185 struct umtx_pi *pi; 1186 1187 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); 1188 TAILQ_INIT(&pi->pi_blocked); 1189 atomic_add_int(&umtx_pi_allocated, 1); 1190 return (pi); 1191 } 1192 1193 static inline void 1194 umtx_pi_free(struct umtx_pi *pi) 1195 { 1196 uma_zfree(umtx_pi_zone, pi); 1197 atomic_add_int(&umtx_pi_allocated, -1); 1198 } 1199 1200 /* 1201 * Adjust the thread's position on a pi_state after its priority has been 1202 * changed. 1203 */ 1204 static int 1205 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) 1206 { 1207 struct umtx_q *uq, *uq1, *uq2; 1208 struct thread *td1; 1209 1210 mtx_assert(&umtx_lock, MA_OWNED); 1211 if (pi == NULL) 1212 return (0); 1213 1214 uq = td->td_umtxq; 1215 1216 /* 1217 * Check if the thread needs to be moved on the blocked chain. 1218 * It needs to be moved if either its priority is lower than 1219 * the previous thread or higher than the next thread. 1220 */ 1221 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); 1222 uq2 = TAILQ_NEXT(uq, uq_lockq); 1223 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || 1224 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { 1225 /* 1226 * Remove thread from blocked chain and determine where 1227 * it should be moved to. 1228 */ 1229 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1230 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1231 td1 = uq1->uq_thread; 1232 MPASS(td1->td_proc->p_magic == P_MAGIC); 1233 if (UPRI(td1) > UPRI(td)) 1234 break; 1235 } 1236 1237 if (uq1 == NULL) 1238 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1239 else 1240 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1241 } 1242 return (1); 1243 } 1244 1245 /* 1246 * Propagate priority when a thread is blocked on POSIX 1247 * PI mutex. 1248 */ 1249 static void 1250 umtx_propagate_priority(struct thread *td) 1251 { 1252 struct umtx_q *uq; 1253 struct umtx_pi *pi; 1254 int pri; 1255 1256 mtx_assert(&umtx_lock, MA_OWNED); 1257 pri = UPRI(td); 1258 uq = td->td_umtxq; 1259 pi = uq->uq_pi_blocked; 1260 if (pi == NULL) 1261 return; 1262 1263 for (;;) { 1264 td = pi->pi_owner; 1265 if (td == NULL) 1266 return; 1267 1268 MPASS(td->td_proc != NULL); 1269 MPASS(td->td_proc->p_magic == P_MAGIC); 1270 1271 if (UPRI(td) <= pri) 1272 return; 1273 1274 thread_lock(td); 1275 sched_lend_user_prio(td, pri); 1276 thread_unlock(td); 1277 1278 /* 1279 * Pick up the lock that td is blocked on. 1280 */ 1281 uq = td->td_umtxq; 1282 pi = uq->uq_pi_blocked; 1283 /* Resort td on the list if needed. */ 1284 if (!umtx_pi_adjust_thread(pi, td)) 1285 break; 1286 } 1287 } 1288 1289 /* 1290 * Unpropagate priority for a PI mutex when a thread blocked on 1291 * it is interrupted by signal or resumed by others. 1292 */ 1293 static void 1294 umtx_unpropagate_priority(struct umtx_pi *pi) 1295 { 1296 struct umtx_q *uq, *uq_owner; 1297 struct umtx_pi *pi2; 1298 int pri, oldpri; 1299 1300 mtx_assert(&umtx_lock, MA_OWNED); 1301 1302 while (pi != NULL && pi->pi_owner != NULL) { 1303 pri = PRI_MAX; 1304 uq_owner = pi->pi_owner->td_umtxq; 1305 1306 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { 1307 uq = TAILQ_FIRST(&pi2->pi_blocked); 1308 if (uq != NULL) { 1309 if (pri > UPRI(uq->uq_thread)) 1310 pri = UPRI(uq->uq_thread); 1311 } 1312 } 1313 1314 if (pri > uq_owner->uq_inherited_pri) 1315 pri = uq_owner->uq_inherited_pri; 1316 thread_lock(pi->pi_owner); 1317 oldpri = pi->pi_owner->td_user_pri; 1318 sched_unlend_user_prio(pi->pi_owner, pri); 1319 thread_unlock(pi->pi_owner); 1320 umtx_pi_adjust_locked(pi->pi_owner, oldpri); 1321 pi = uq_owner->uq_pi_blocked; 1322 } 1323 } 1324 1325 /* 1326 * Insert a PI mutex into owned list. 1327 */ 1328 static void 1329 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) 1330 { 1331 struct umtx_q *uq_owner; 1332 1333 uq_owner = owner->td_umtxq; 1334 mtx_assert(&umtx_lock, MA_OWNED); 1335 if (pi->pi_owner != NULL) 1336 panic("pi_ower != NULL"); 1337 pi->pi_owner = owner; 1338 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); 1339 } 1340 1341 /* 1342 * Claim ownership of a PI mutex. 1343 */ 1344 static int 1345 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) 1346 { 1347 struct umtx_q *uq, *uq_owner; 1348 1349 uq_owner = owner->td_umtxq; 1350 mtx_lock_spin(&umtx_lock); 1351 if (pi->pi_owner == owner) { 1352 mtx_unlock_spin(&umtx_lock); 1353 return (0); 1354 } 1355 1356 if (pi->pi_owner != NULL) { 1357 /* 1358 * userland may have already messed the mutex, sigh. 1359 */ 1360 mtx_unlock_spin(&umtx_lock); 1361 return (EPERM); 1362 } 1363 umtx_pi_setowner(pi, owner); 1364 uq = TAILQ_FIRST(&pi->pi_blocked); 1365 if (uq != NULL) { 1366 int pri; 1367 1368 pri = UPRI(uq->uq_thread); 1369 thread_lock(owner); 1370 if (pri < UPRI(owner)) 1371 sched_lend_user_prio(owner, pri); 1372 thread_unlock(owner); 1373 } 1374 mtx_unlock_spin(&umtx_lock); 1375 return (0); 1376 } 1377 1378 static void 1379 umtx_pi_adjust_locked(struct thread *td, u_char oldpri) 1380 { 1381 struct umtx_q *uq; 1382 struct umtx_pi *pi; 1383 1384 uq = td->td_umtxq; 1385 1386 mtx_assert(&umtx_lock, MA_OWNED); 1387 MPASS(TD_ON_UPILOCK(td)); 1388 1389 /* 1390 * Pick up the lock that td is blocked on. 1391 */ 1392 pi = uq->uq_pi_blocked; 1393 MPASS(pi != NULL); 1394 1395 /* Resort the turnstile on the list. */ 1396 if (!umtx_pi_adjust_thread(pi, td)) 1397 return; 1398 1399 /* 1400 * If our priority was lowered and we are at the head of the 1401 * turnstile, then propagate our new priority up the chain. 1402 */ 1403 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri) 1404 umtx_propagate_priority(td); 1405 } 1406 1407 /* 1408 * Adjust a thread's order position in its blocked PI mutex, 1409 * this may result new priority propagating process. 1410 */ 1411 void 1412 umtx_pi_adjust(struct thread *td, u_char oldpri) 1413 { 1414 struct umtx_q *uq; 1415 struct umtx_pi *pi; 1416 1417 uq = td->td_umtxq; 1418 mtx_lock_spin(&umtx_lock); 1419 /* 1420 * Pick up the lock that td is blocked on. 1421 */ 1422 pi = uq->uq_pi_blocked; 1423 if (pi != NULL) 1424 umtx_pi_adjust_locked(td, oldpri); 1425 mtx_unlock_spin(&umtx_lock); 1426 } 1427 1428 /* 1429 * Sleep on a PI mutex. 1430 */ 1431 static int 1432 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, 1433 uint32_t owner, const char *wmesg, int timo) 1434 { 1435 struct umtxq_chain *uc; 1436 struct thread *td, *td1; 1437 struct umtx_q *uq1; 1438 int pri; 1439 int error = 0; 1440 1441 td = uq->uq_thread; 1442 KASSERT(td == curthread, ("inconsistent uq_thread")); 1443 uc = umtxq_getchain(&uq->uq_key); 1444 UMTXQ_LOCKED_ASSERT(uc); 1445 umtxq_insert(uq); 1446 if (pi->pi_owner == NULL) { 1447 /* XXX 1448 * Current, We only support process private PI-mutex, 1449 * non-contended PI-mutexes are locked in userland. 1450 * Process shared PI-mutex should always be initialized 1451 * by kernel and be registered in kernel, locking should 1452 * always be done by kernel to avoid security problems. 1453 * For process private PI-mutex, we can find owner 1454 * thread and boost its priority safely. 1455 */ 1456 PROC_LOCK(curproc); 1457 td1 = thread_find(curproc, owner); 1458 mtx_lock_spin(&umtx_lock); 1459 if (td1 != NULL && pi->pi_owner == NULL) { 1460 uq1 = td1->td_umtxq; 1461 umtx_pi_setowner(pi, td1); 1462 } 1463 PROC_UNLOCK(curproc); 1464 } else { 1465 mtx_lock_spin(&umtx_lock); 1466 } 1467 1468 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1469 pri = UPRI(uq1->uq_thread); 1470 if (pri > UPRI(td)) 1471 break; 1472 } 1473 1474 if (uq1 != NULL) 1475 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1476 else 1477 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1478 1479 uq->uq_pi_blocked = pi; 1480 td->td_flags |= TDF_UPIBLOCKED; 1481 mtx_unlock_spin(&umtx_lock); 1482 umtxq_unlock(&uq->uq_key); 1483 1484 mtx_lock_spin(&umtx_lock); 1485 umtx_propagate_priority(td); 1486 mtx_unlock_spin(&umtx_lock); 1487 1488 umtxq_lock(&uq->uq_key); 1489 if (uq->uq_flags & UQF_UMTXQ) { 1490 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 1491 if (error == EWOULDBLOCK) 1492 error = ETIMEDOUT; 1493 if (uq->uq_flags & UQF_UMTXQ) { 1494 umtxq_busy(&uq->uq_key); 1495 umtxq_remove(uq); 1496 umtxq_unbusy(&uq->uq_key); 1497 } 1498 } 1499 umtxq_unlock(&uq->uq_key); 1500 1501 mtx_lock_spin(&umtx_lock); 1502 uq->uq_pi_blocked = NULL; 1503 thread_lock(td); 1504 td->td_flags &= ~TDF_UPIBLOCKED; 1505 thread_unlock(td); 1506 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1507 umtx_unpropagate_priority(pi); 1508 mtx_unlock_spin(&umtx_lock); 1509 1510 umtxq_lock(&uq->uq_key); 1511 1512 return (error); 1513 } 1514 1515 /* 1516 * Add reference count for a PI mutex. 1517 */ 1518 static void 1519 umtx_pi_ref(struct umtx_pi *pi) 1520 { 1521 struct umtxq_chain *uc; 1522 1523 uc = umtxq_getchain(&pi->pi_key); 1524 UMTXQ_LOCKED_ASSERT(uc); 1525 pi->pi_refcount++; 1526 } 1527 1528 /* 1529 * Decrease reference count for a PI mutex, if the counter 1530 * is decreased to zero, its memory space is freed. 1531 */ 1532 static void 1533 umtx_pi_unref(struct umtx_pi *pi) 1534 { 1535 struct umtxq_chain *uc; 1536 int free = 0; 1537 1538 uc = umtxq_getchain(&pi->pi_key); 1539 UMTXQ_LOCKED_ASSERT(uc); 1540 KASSERT(pi->pi_refcount > 0, ("invalid reference count")); 1541 if (--pi->pi_refcount == 0) { 1542 mtx_lock_spin(&umtx_lock); 1543 if (pi->pi_owner != NULL) { 1544 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, 1545 pi, pi_link); 1546 pi->pi_owner = NULL; 1547 } 1548 KASSERT(TAILQ_EMPTY(&pi->pi_blocked), 1549 ("blocked queue not empty")); 1550 mtx_unlock_spin(&umtx_lock); 1551 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); 1552 free = 1; 1553 } 1554 if (free) 1555 umtx_pi_free(pi); 1556 } 1557 1558 /* 1559 * Find a PI mutex in hash table. 1560 */ 1561 static struct umtx_pi * 1562 umtx_pi_lookup(struct umtx_key *key) 1563 { 1564 struct umtxq_chain *uc; 1565 struct umtx_pi *pi; 1566 1567 uc = umtxq_getchain(key); 1568 UMTXQ_LOCKED_ASSERT(uc); 1569 1570 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { 1571 if (umtx_key_match(&pi->pi_key, key)) { 1572 return (pi); 1573 } 1574 } 1575 return (NULL); 1576 } 1577 1578 /* 1579 * Insert a PI mutex into hash table. 1580 */ 1581 static inline void 1582 umtx_pi_insert(struct umtx_pi *pi) 1583 { 1584 struct umtxq_chain *uc; 1585 1586 uc = umtxq_getchain(&pi->pi_key); 1587 UMTXQ_LOCKED_ASSERT(uc); 1588 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); 1589 } 1590 1591 /* 1592 * Lock a PI mutex. 1593 */ 1594 static int 1595 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1596 int try) 1597 { 1598 struct umtx_q *uq; 1599 struct umtx_pi *pi, *new_pi; 1600 uint32_t id, owner, old; 1601 int error; 1602 1603 id = td->td_tid; 1604 uq = td->td_umtxq; 1605 1606 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1607 &uq->uq_key)) != 0) 1608 return (error); 1609 umtxq_lock(&uq->uq_key); 1610 pi = umtx_pi_lookup(&uq->uq_key); 1611 if (pi == NULL) { 1612 new_pi = umtx_pi_alloc(M_NOWAIT); 1613 if (new_pi == NULL) { 1614 umtxq_unlock(&uq->uq_key); 1615 new_pi = umtx_pi_alloc(M_WAITOK); 1616 new_pi->pi_key = uq->uq_key; 1617 umtxq_lock(&uq->uq_key); 1618 pi = umtx_pi_lookup(&uq->uq_key); 1619 if (pi != NULL) { 1620 umtx_pi_free(new_pi); 1621 new_pi = NULL; 1622 } 1623 } 1624 if (new_pi != NULL) { 1625 new_pi->pi_key = uq->uq_key; 1626 umtx_pi_insert(new_pi); 1627 pi = new_pi; 1628 } 1629 } 1630 umtx_pi_ref(pi); 1631 umtxq_unlock(&uq->uq_key); 1632 1633 /* 1634 * Care must be exercised when dealing with umtx structure. It 1635 * can fault on any access. 1636 */ 1637 for (;;) { 1638 /* 1639 * Try the uncontested case. This should be done in userland. 1640 */ 1641 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1642 1643 /* The acquire succeeded. */ 1644 if (owner == UMUTEX_UNOWNED) { 1645 error = 0; 1646 break; 1647 } 1648 1649 /* The address was invalid. */ 1650 if (owner == -1) { 1651 error = EFAULT; 1652 break; 1653 } 1654 1655 /* If no one owns it but it is contested try to acquire it. */ 1656 if (owner == UMUTEX_CONTESTED) { 1657 owner = casuword32(&m->m_owner, 1658 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1659 1660 if (owner == UMUTEX_CONTESTED) { 1661 umtxq_lock(&uq->uq_key); 1662 error = umtx_pi_claim(pi, td); 1663 umtxq_unlock(&uq->uq_key); 1664 break; 1665 } 1666 1667 /* The address was invalid. */ 1668 if (owner == -1) { 1669 error = EFAULT; 1670 break; 1671 } 1672 1673 /* If this failed the lock has changed, restart. */ 1674 continue; 1675 } 1676 1677 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1678 (owner & ~UMUTEX_CONTESTED) == id) { 1679 error = EDEADLK; 1680 break; 1681 } 1682 1683 if (try != 0) { 1684 error = EBUSY; 1685 break; 1686 } 1687 1688 /* 1689 * If we caught a signal, we have retried and now 1690 * exit immediately. 1691 */ 1692 if (error != 0) 1693 break; 1694 1695 umtxq_lock(&uq->uq_key); 1696 umtxq_busy(&uq->uq_key); 1697 umtxq_unlock(&uq->uq_key); 1698 1699 /* 1700 * Set the contested bit so that a release in user space 1701 * knows to use the system call for unlock. If this fails 1702 * either some one else has acquired the lock or it has been 1703 * released. 1704 */ 1705 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1706 1707 /* The address was invalid. */ 1708 if (old == -1) { 1709 umtxq_lock(&uq->uq_key); 1710 umtxq_unbusy(&uq->uq_key); 1711 umtxq_unlock(&uq->uq_key); 1712 error = EFAULT; 1713 break; 1714 } 1715 1716 umtxq_lock(&uq->uq_key); 1717 umtxq_unbusy(&uq->uq_key); 1718 /* 1719 * We set the contested bit, sleep. Otherwise the lock changed 1720 * and we need to retry or we lost a race to the thread 1721 * unlocking the umtx. 1722 */ 1723 if (old == owner) 1724 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, 1725 "umtxpi", timo); 1726 umtxq_unlock(&uq->uq_key); 1727 } 1728 1729 umtxq_lock(&uq->uq_key); 1730 umtx_pi_unref(pi); 1731 umtxq_unlock(&uq->uq_key); 1732 1733 umtx_key_release(&uq->uq_key); 1734 return (error); 1735 } 1736 1737 /* 1738 * Unlock a PI mutex. 1739 */ 1740 static int 1741 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) 1742 { 1743 struct umtx_key key; 1744 struct umtx_q *uq_first, *uq_first2, *uq_me; 1745 struct umtx_pi *pi, *pi2; 1746 uint32_t owner, old, id; 1747 int error; 1748 int count; 1749 int pri; 1750 1751 id = td->td_tid; 1752 /* 1753 * Make sure we own this mtx. 1754 */ 1755 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1756 if (owner == -1) 1757 return (EFAULT); 1758 1759 if ((owner & ~UMUTEX_CONTESTED) != id) 1760 return (EPERM); 1761 1762 /* This should be done in userland */ 1763 if ((owner & UMUTEX_CONTESTED) == 0) { 1764 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1765 if (old == -1) 1766 return (EFAULT); 1767 if (old == owner) 1768 return (0); 1769 owner = old; 1770 } 1771 1772 /* We should only ever be in here for contested locks */ 1773 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1774 &key)) != 0) 1775 return (error); 1776 1777 umtxq_lock(&key); 1778 umtxq_busy(&key); 1779 count = umtxq_count_pi(&key, &uq_first); 1780 if (uq_first != NULL) { 1781 pi = uq_first->uq_pi_blocked; 1782 if (pi->pi_owner != curthread) { 1783 umtxq_unbusy(&key); 1784 umtxq_unlock(&key); 1785 /* userland messed the mutex */ 1786 return (EPERM); 1787 } 1788 uq_me = curthread->td_umtxq; 1789 mtx_lock_spin(&umtx_lock); 1790 pi->pi_owner = NULL; 1791 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); 1792 uq_first = TAILQ_FIRST(&pi->pi_blocked); 1793 pri = PRI_MAX; 1794 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { 1795 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); 1796 if (uq_first2 != NULL) { 1797 if (pri > UPRI(uq_first2->uq_thread)) 1798 pri = UPRI(uq_first2->uq_thread); 1799 } 1800 } 1801 thread_lock(curthread); 1802 sched_unlend_user_prio(curthread, pri); 1803 thread_unlock(curthread); 1804 mtx_unlock_spin(&umtx_lock); 1805 } 1806 umtxq_unlock(&key); 1807 1808 /* 1809 * When unlocking the umtx, it must be marked as unowned if 1810 * there is zero or one thread only waiting for it. 1811 * Otherwise, it must be marked as contested. 1812 */ 1813 old = casuword32(&m->m_owner, owner, 1814 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1815 1816 umtxq_lock(&key); 1817 if (uq_first != NULL) 1818 umtxq_signal_thread(uq_first); 1819 umtxq_unbusy(&key); 1820 umtxq_unlock(&key); 1821 umtx_key_release(&key); 1822 if (old == -1) 1823 return (EFAULT); 1824 if (old != owner) 1825 return (EINVAL); 1826 return (0); 1827 } 1828 1829 /* 1830 * Lock a PP mutex. 1831 */ 1832 static int 1833 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1834 int try) 1835 { 1836 struct umtx_q *uq, *uq2; 1837 struct umtx_pi *pi; 1838 uint32_t ceiling; 1839 uint32_t owner, id; 1840 int error, pri, old_inherited_pri, su; 1841 1842 id = td->td_tid; 1843 uq = td->td_umtxq; 1844 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 1845 &uq->uq_key)) != 0) 1846 return (error); 1847 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 1848 for (;;) { 1849 old_inherited_pri = uq->uq_inherited_pri; 1850 umtxq_lock(&uq->uq_key); 1851 umtxq_busy(&uq->uq_key); 1852 umtxq_unlock(&uq->uq_key); 1853 1854 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); 1855 if (ceiling > RTP_PRIO_MAX) { 1856 error = EINVAL; 1857 goto out; 1858 } 1859 1860 mtx_lock_spin(&umtx_lock); 1861 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { 1862 mtx_unlock_spin(&umtx_lock); 1863 error = EINVAL; 1864 goto out; 1865 } 1866 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { 1867 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; 1868 thread_lock(td); 1869 if (uq->uq_inherited_pri < UPRI(td)) 1870 sched_lend_user_prio(td, uq->uq_inherited_pri); 1871 thread_unlock(td); 1872 } 1873 mtx_unlock_spin(&umtx_lock); 1874 1875 owner = casuword32(&m->m_owner, 1876 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1877 1878 if (owner == UMUTEX_CONTESTED) { 1879 error = 0; 1880 break; 1881 } 1882 1883 /* The address was invalid. */ 1884 if (owner == -1) { 1885 error = EFAULT; 1886 break; 1887 } 1888 1889 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1890 (owner & ~UMUTEX_CONTESTED) == id) { 1891 error = EDEADLK; 1892 break; 1893 } 1894 1895 if (try != 0) { 1896 error = EBUSY; 1897 break; 1898 } 1899 1900 /* 1901 * If we caught a signal, we have retried and now 1902 * exit immediately. 1903 */ 1904 if (error != 0) 1905 break; 1906 1907 umtxq_lock(&uq->uq_key); 1908 umtxq_insert(uq); 1909 umtxq_unbusy(&uq->uq_key); 1910 error = umtxq_sleep(uq, "umtxpp", timo); 1911 umtxq_remove(uq); 1912 umtxq_unlock(&uq->uq_key); 1913 1914 mtx_lock_spin(&umtx_lock); 1915 uq->uq_inherited_pri = old_inherited_pri; 1916 pri = PRI_MAX; 1917 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 1918 uq2 = TAILQ_FIRST(&pi->pi_blocked); 1919 if (uq2 != NULL) { 1920 if (pri > UPRI(uq2->uq_thread)) 1921 pri = UPRI(uq2->uq_thread); 1922 } 1923 } 1924 if (pri > uq->uq_inherited_pri) 1925 pri = uq->uq_inherited_pri; 1926 thread_lock(td); 1927 sched_unlend_user_prio(td, pri); 1928 thread_unlock(td); 1929 mtx_unlock_spin(&umtx_lock); 1930 } 1931 1932 if (error != 0) { 1933 mtx_lock_spin(&umtx_lock); 1934 uq->uq_inherited_pri = old_inherited_pri; 1935 pri = PRI_MAX; 1936 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 1937 uq2 = TAILQ_FIRST(&pi->pi_blocked); 1938 if (uq2 != NULL) { 1939 if (pri > UPRI(uq2->uq_thread)) 1940 pri = UPRI(uq2->uq_thread); 1941 } 1942 } 1943 if (pri > uq->uq_inherited_pri) 1944 pri = uq->uq_inherited_pri; 1945 thread_lock(td); 1946 sched_unlend_user_prio(td, pri); 1947 thread_unlock(td); 1948 mtx_unlock_spin(&umtx_lock); 1949 } 1950 1951 out: 1952 umtxq_lock(&uq->uq_key); 1953 umtxq_unbusy(&uq->uq_key); 1954 umtxq_unlock(&uq->uq_key); 1955 umtx_key_release(&uq->uq_key); 1956 return (error); 1957 } 1958 1959 /* 1960 * Unlock a PP mutex. 1961 */ 1962 static int 1963 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) 1964 { 1965 struct umtx_key key; 1966 struct umtx_q *uq, *uq2; 1967 struct umtx_pi *pi; 1968 uint32_t owner, id; 1969 uint32_t rceiling; 1970 int error, pri, new_inherited_pri, su; 1971 1972 id = td->td_tid; 1973 uq = td->td_umtxq; 1974 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 1975 1976 /* 1977 * Make sure we own this mtx. 1978 */ 1979 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1980 if (owner == -1) 1981 return (EFAULT); 1982 1983 if ((owner & ~UMUTEX_CONTESTED) != id) 1984 return (EPERM); 1985 1986 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); 1987 if (error != 0) 1988 return (error); 1989 1990 if (rceiling == -1) 1991 new_inherited_pri = PRI_MAX; 1992 else { 1993 rceiling = RTP_PRIO_MAX - rceiling; 1994 if (rceiling > RTP_PRIO_MAX) 1995 return (EINVAL); 1996 new_inherited_pri = PRI_MIN_REALTIME + rceiling; 1997 } 1998 1999 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2000 &key)) != 0) 2001 return (error); 2002 umtxq_lock(&key); 2003 umtxq_busy(&key); 2004 umtxq_unlock(&key); 2005 /* 2006 * For priority protected mutex, always set unlocked state 2007 * to UMUTEX_CONTESTED, so that userland always enters kernel 2008 * to lock the mutex, it is necessary because thread priority 2009 * has to be adjusted for such mutex. 2010 */ 2011 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2012 UMUTEX_CONTESTED); 2013 2014 umtxq_lock(&key); 2015 if (error == 0) 2016 umtxq_signal(&key, 1); 2017 umtxq_unbusy(&key); 2018 umtxq_unlock(&key); 2019 2020 if (error == -1) 2021 error = EFAULT; 2022 else { 2023 mtx_lock_spin(&umtx_lock); 2024 if (su != 0) 2025 uq->uq_inherited_pri = new_inherited_pri; 2026 pri = PRI_MAX; 2027 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2028 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2029 if (uq2 != NULL) { 2030 if (pri > UPRI(uq2->uq_thread)) 2031 pri = UPRI(uq2->uq_thread); 2032 } 2033 } 2034 if (pri > uq->uq_inherited_pri) 2035 pri = uq->uq_inherited_pri; 2036 thread_lock(td); 2037 sched_unlend_user_prio(td, pri); 2038 thread_unlock(td); 2039 mtx_unlock_spin(&umtx_lock); 2040 } 2041 umtx_key_release(&key); 2042 return (error); 2043 } 2044 2045 static int 2046 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, 2047 uint32_t *old_ceiling) 2048 { 2049 struct umtx_q *uq; 2050 uint32_t save_ceiling; 2051 uint32_t owner, id; 2052 uint32_t flags; 2053 int error; 2054 2055 flags = fuword32(&m->m_flags); 2056 if ((flags & UMUTEX_PRIO_PROTECT) == 0) 2057 return (EINVAL); 2058 if (ceiling > RTP_PRIO_MAX) 2059 return (EINVAL); 2060 id = td->td_tid; 2061 uq = td->td_umtxq; 2062 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2063 &uq->uq_key)) != 0) 2064 return (error); 2065 for (;;) { 2066 umtxq_lock(&uq->uq_key); 2067 umtxq_busy(&uq->uq_key); 2068 umtxq_unlock(&uq->uq_key); 2069 2070 save_ceiling = fuword32(&m->m_ceilings[0]); 2071 2072 owner = casuword32(&m->m_owner, 2073 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 2074 2075 if (owner == UMUTEX_CONTESTED) { 2076 suword32(&m->m_ceilings[0], ceiling); 2077 suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2078 UMUTEX_CONTESTED); 2079 error = 0; 2080 break; 2081 } 2082 2083 /* The address was invalid. */ 2084 if (owner == -1) { 2085 error = EFAULT; 2086 break; 2087 } 2088 2089 if ((owner & ~UMUTEX_CONTESTED) == id) { 2090 suword32(&m->m_ceilings[0], ceiling); 2091 error = 0; 2092 break; 2093 } 2094 2095 /* 2096 * If we caught a signal, we have retried and now 2097 * exit immediately. 2098 */ 2099 if (error != 0) 2100 break; 2101 2102 /* 2103 * We set the contested bit, sleep. Otherwise the lock changed 2104 * and we need to retry or we lost a race to the thread 2105 * unlocking the umtx. 2106 */ 2107 umtxq_lock(&uq->uq_key); 2108 umtxq_insert(uq); 2109 umtxq_unbusy(&uq->uq_key); 2110 error = umtxq_sleep(uq, "umtxpp", 0); 2111 umtxq_remove(uq); 2112 umtxq_unlock(&uq->uq_key); 2113 } 2114 umtxq_lock(&uq->uq_key); 2115 if (error == 0) 2116 umtxq_signal(&uq->uq_key, INT_MAX); 2117 umtxq_unbusy(&uq->uq_key); 2118 umtxq_unlock(&uq->uq_key); 2119 umtx_key_release(&uq->uq_key); 2120 if (error == 0 && old_ceiling != NULL) 2121 suword32(old_ceiling, save_ceiling); 2122 return (error); 2123 } 2124 2125 static int 2126 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo, 2127 int try) 2128 { 2129 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2130 case 0: 2131 return (_do_lock_normal(td, m, flags, timo, try)); 2132 case UMUTEX_PRIO_INHERIT: 2133 return (_do_lock_pi(td, m, flags, timo, try)); 2134 case UMUTEX_PRIO_PROTECT: 2135 return (_do_lock_pp(td, m, flags, timo, try)); 2136 } 2137 return (EINVAL); 2138 } 2139 2140 /* 2141 * Lock a userland POSIX mutex. 2142 */ 2143 static int 2144 do_lock_umutex(struct thread *td, struct umutex *m, 2145 struct timespec *timeout, int try) 2146 { 2147 struct timespec ts, ts2, ts3; 2148 struct timeval tv; 2149 uint32_t flags; 2150 int error; 2151 2152 flags = fuword32(&m->m_flags); 2153 if (flags == -1) 2154 return (EFAULT); 2155 2156 if (timeout == NULL) { 2157 error = _do_lock_umutex(td, m, flags, 0, try); 2158 /* Mutex locking is restarted if it is interrupted. */ 2159 if (error == EINTR) 2160 error = ERESTART; 2161 } else { 2162 getnanouptime(&ts); 2163 timespecadd(&ts, timeout); 2164 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2165 for (;;) { 2166 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), try); 2167 if (error != ETIMEDOUT) 2168 break; 2169 getnanouptime(&ts2); 2170 if (timespeccmp(&ts2, &ts, >=)) { 2171 error = ETIMEDOUT; 2172 break; 2173 } 2174 ts3 = ts; 2175 timespecsub(&ts3, &ts2); 2176 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2177 } 2178 /* Timed-locking is not restarted. */ 2179 if (error == ERESTART) 2180 error = EINTR; 2181 } 2182 return (error); 2183 } 2184 2185 /* 2186 * Unlock a userland POSIX mutex. 2187 */ 2188 static int 2189 do_unlock_umutex(struct thread *td, struct umutex *m) 2190 { 2191 uint32_t flags; 2192 2193 flags = fuword32(&m->m_flags); 2194 if (flags == -1) 2195 return (EFAULT); 2196 2197 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2198 case 0: 2199 return (do_unlock_normal(td, m, flags)); 2200 case UMUTEX_PRIO_INHERIT: 2201 return (do_unlock_pi(td, m, flags)); 2202 case UMUTEX_PRIO_PROTECT: 2203 return (do_unlock_pp(td, m, flags)); 2204 } 2205 2206 return (EINVAL); 2207 } 2208 2209 static int 2210 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2211 struct timespec *timeout, u_long wflags) 2212 { 2213 struct umtx_q *uq; 2214 struct timeval tv; 2215 struct timespec cts, ets, tts; 2216 uint32_t flags; 2217 int error; 2218 2219 uq = td->td_umtxq; 2220 flags = fuword32(&cv->c_flags); 2221 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2222 if (error != 0) 2223 return (error); 2224 umtxq_lock(&uq->uq_key); 2225 umtxq_busy(&uq->uq_key); 2226 umtxq_insert(uq); 2227 umtxq_unlock(&uq->uq_key); 2228 2229 /* 2230 * The magic thing is we should set c_has_waiters to 1 before 2231 * releasing user mutex. 2232 */ 2233 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); 2234 2235 umtxq_lock(&uq->uq_key); 2236 umtxq_unbusy(&uq->uq_key); 2237 umtxq_unlock(&uq->uq_key); 2238 2239 error = do_unlock_umutex(td, m); 2240 2241 umtxq_lock(&uq->uq_key); 2242 if (error == 0) { 2243 if ((wflags & UMTX_CHECK_UNPARKING) && 2244 (td->td_pflags & TDP_WAKEUP)) { 2245 td->td_pflags &= ~TDP_WAKEUP; 2246 error = EINTR; 2247 } else if (timeout == NULL) { 2248 error = umtxq_sleep(uq, "ucond", 0); 2249 } else { 2250 getnanouptime(&ets); 2251 timespecadd(&ets, timeout); 2252 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2253 for (;;) { 2254 error = umtxq_sleep(uq, "ucond", tvtohz(&tv)); 2255 if (error != ETIMEDOUT) 2256 break; 2257 getnanouptime(&cts); 2258 if (timespeccmp(&cts, &ets, >=)) { 2259 error = ETIMEDOUT; 2260 break; 2261 } 2262 tts = ets; 2263 timespecsub(&tts, &cts); 2264 TIMESPEC_TO_TIMEVAL(&tv, &tts); 2265 } 2266 } 2267 } 2268 2269 if (error != 0) { 2270 if ((uq->uq_flags & UQF_UMTXQ) == 0) { 2271 /* 2272 * If we concurrently got do_cv_signal()d 2273 * and we got an error or UNIX signals or a timeout, 2274 * then, perform another umtxq_signal to avoid 2275 * consuming the wakeup. This may cause supurious 2276 * wakeup for another thread which was just queued, 2277 * but SUSV3 explicitly allows supurious wakeup to 2278 * occur, and indeed a kernel based implementation 2279 * can not avoid it. 2280 */ 2281 if (!umtxq_signal(&uq->uq_key, 1)) 2282 error = 0; 2283 } 2284 if (error == ERESTART) 2285 error = EINTR; 2286 } 2287 umtxq_remove(uq); 2288 umtxq_unlock(&uq->uq_key); 2289 umtx_key_release(&uq->uq_key); 2290 return (error); 2291 } 2292 2293 /* 2294 * Signal a userland condition variable. 2295 */ 2296 static int 2297 do_cv_signal(struct thread *td, struct ucond *cv) 2298 { 2299 struct umtx_key key; 2300 int error, cnt, nwake; 2301 uint32_t flags; 2302 2303 flags = fuword32(&cv->c_flags); 2304 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2305 return (error); 2306 umtxq_lock(&key); 2307 umtxq_busy(&key); 2308 cnt = umtxq_count(&key); 2309 nwake = umtxq_signal(&key, 1); 2310 if (cnt <= nwake) { 2311 umtxq_unlock(&key); 2312 error = suword32( 2313 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2314 umtxq_lock(&key); 2315 } 2316 umtxq_unbusy(&key); 2317 umtxq_unlock(&key); 2318 umtx_key_release(&key); 2319 return (error); 2320 } 2321 2322 static int 2323 do_cv_broadcast(struct thread *td, struct ucond *cv) 2324 { 2325 struct umtx_key key; 2326 int error; 2327 uint32_t flags; 2328 2329 flags = fuword32(&cv->c_flags); 2330 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2331 return (error); 2332 2333 umtxq_lock(&key); 2334 umtxq_busy(&key); 2335 umtxq_signal(&key, INT_MAX); 2336 umtxq_unlock(&key); 2337 2338 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2339 2340 umtxq_lock(&key); 2341 umtxq_unbusy(&key); 2342 umtxq_unlock(&key); 2343 2344 umtx_key_release(&key); 2345 return (error); 2346 } 2347 2348 int 2349 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 2350 /* struct umtx *umtx */ 2351 { 2352 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0); 2353 } 2354 2355 int 2356 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 2357 /* struct umtx *umtx */ 2358 { 2359 return do_unlock_umtx(td, uap->umtx, td->td_tid); 2360 } 2361 2362 static int 2363 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap) 2364 { 2365 struct timespec *ts, timeout; 2366 int error; 2367 2368 /* Allow a null timespec (wait forever). */ 2369 if (uap->uaddr2 == NULL) 2370 ts = NULL; 2371 else { 2372 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2373 if (error != 0) 2374 return (error); 2375 if (timeout.tv_nsec >= 1000000000 || 2376 timeout.tv_nsec < 0) { 2377 return (EINVAL); 2378 } 2379 ts = &timeout; 2380 } 2381 return (do_lock_umtx(td, uap->obj, uap->val, ts)); 2382 } 2383 2384 static int 2385 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap) 2386 { 2387 return (do_unlock_umtx(td, uap->obj, uap->val)); 2388 } 2389 2390 static int 2391 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) 2392 { 2393 struct timespec *ts, timeout; 2394 int error; 2395 2396 if (uap->uaddr2 == NULL) 2397 ts = NULL; 2398 else { 2399 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2400 if (error != 0) 2401 return (error); 2402 if (timeout.tv_nsec >= 1000000000 || 2403 timeout.tv_nsec < 0) 2404 return (EINVAL); 2405 ts = &timeout; 2406 } 2407 return do_wait(td, uap->obj, uap->val, ts, 0); 2408 } 2409 2410 static int 2411 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 2412 { 2413 return (kern_umtx_wake(td, uap->obj, uap->val)); 2414 } 2415 2416 static int 2417 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 2418 { 2419 struct timespec *ts, timeout; 2420 int error; 2421 2422 /* Allow a null timespec (wait forever). */ 2423 if (uap->uaddr2 == NULL) 2424 ts = NULL; 2425 else { 2426 error = copyin(uap->uaddr2, &timeout, 2427 sizeof(timeout)); 2428 if (error != 0) 2429 return (error); 2430 if (timeout.tv_nsec >= 1000000000 || 2431 timeout.tv_nsec < 0) { 2432 return (EINVAL); 2433 } 2434 ts = &timeout; 2435 } 2436 return do_lock_umutex(td, uap->obj, ts, 0); 2437 } 2438 2439 static int 2440 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) 2441 { 2442 return do_lock_umutex(td, uap->obj, NULL, 1); 2443 } 2444 2445 static int 2446 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) 2447 { 2448 return do_unlock_umutex(td, uap->obj); 2449 } 2450 2451 static int 2452 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) 2453 { 2454 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); 2455 } 2456 2457 static int 2458 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) 2459 { 2460 struct timespec *ts, timeout; 2461 int error; 2462 2463 /* Allow a null timespec (wait forever). */ 2464 if (uap->uaddr2 == NULL) 2465 ts = NULL; 2466 else { 2467 error = copyin(uap->uaddr2, &timeout, 2468 sizeof(timeout)); 2469 if (error != 0) 2470 return (error); 2471 if (timeout.tv_nsec >= 1000000000 || 2472 timeout.tv_nsec < 0) { 2473 return (EINVAL); 2474 } 2475 ts = &timeout; 2476 } 2477 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 2478 } 2479 2480 static int 2481 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) 2482 { 2483 return do_cv_signal(td, uap->obj); 2484 } 2485 2486 static int 2487 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) 2488 { 2489 return do_cv_broadcast(td, uap->obj); 2490 } 2491 2492 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); 2493 2494 static _umtx_op_func op_table[] = { 2495 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */ 2496 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */ 2497 __umtx_op_wait, /* UMTX_OP_WAIT */ 2498 __umtx_op_wake, /* UMTX_OP_WAKE */ 2499 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */ 2500 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */ 2501 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 2502 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 2503 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/ 2504 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 2505 __umtx_op_cv_broadcast /* UMTX_OP_CV_BROADCAST */ 2506 }; 2507 2508 int 2509 _umtx_op(struct thread *td, struct _umtx_op_args *uap) 2510 { 2511 if ((unsigned)uap->op < UMTX_OP_MAX) 2512 return (*op_table[uap->op])(td, uap); 2513 return (EINVAL); 2514 } 2515 2516 #ifdef COMPAT_IA32 2517 int 2518 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap) 2519 /* struct umtx *umtx */ 2520 { 2521 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL)); 2522 } 2523 2524 int 2525 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap) 2526 /* struct umtx *umtx */ 2527 { 2528 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid)); 2529 } 2530 2531 struct timespec32 { 2532 u_int32_t tv_sec; 2533 u_int32_t tv_nsec; 2534 }; 2535 2536 static inline int 2537 copyin_timeout32(void *addr, struct timespec *tsp) 2538 { 2539 struct timespec32 ts32; 2540 int error; 2541 2542 error = copyin(addr, &ts32, sizeof(struct timespec32)); 2543 if (error == 0) { 2544 tsp->tv_sec = ts32.tv_sec; 2545 tsp->tv_nsec = ts32.tv_nsec; 2546 } 2547 return (error); 2548 } 2549 2550 static int 2551 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 2552 { 2553 struct timespec *ts, timeout; 2554 int error; 2555 2556 /* Allow a null timespec (wait forever). */ 2557 if (uap->uaddr2 == NULL) 2558 ts = NULL; 2559 else { 2560 error = copyin_timeout32(uap->uaddr2, &timeout); 2561 if (error != 0) 2562 return (error); 2563 if (timeout.tv_nsec >= 1000000000 || 2564 timeout.tv_nsec < 0) { 2565 return (EINVAL); 2566 } 2567 ts = &timeout; 2568 } 2569 return (do_lock_umtx32(td, uap->obj, uap->val, ts)); 2570 } 2571 2572 static int 2573 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 2574 { 2575 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val)); 2576 } 2577 2578 static int 2579 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 2580 { 2581 struct timespec *ts, timeout; 2582 int error; 2583 2584 if (uap->uaddr2 == NULL) 2585 ts = NULL; 2586 else { 2587 error = copyin_timeout32(uap->uaddr2, &timeout); 2588 if (error != 0) 2589 return (error); 2590 if (timeout.tv_nsec >= 1000000000 || 2591 timeout.tv_nsec < 0) 2592 return (EINVAL); 2593 ts = &timeout; 2594 } 2595 return do_wait(td, uap->obj, uap->val, ts, 1); 2596 } 2597 2598 static int 2599 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 2600 { 2601 struct timespec *ts, timeout; 2602 int error; 2603 2604 /* Allow a null timespec (wait forever). */ 2605 if (uap->uaddr2 == NULL) 2606 ts = NULL; 2607 else { 2608 error = copyin_timeout32(uap->uaddr2, &timeout); 2609 if (error != 0) 2610 return (error); 2611 if (timeout.tv_nsec >= 1000000000 || 2612 timeout.tv_nsec < 0) 2613 return (EINVAL); 2614 ts = &timeout; 2615 } 2616 return do_lock_umutex(td, uap->obj, ts, 0); 2617 } 2618 2619 static int 2620 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 2621 { 2622 struct timespec *ts, timeout; 2623 int error; 2624 2625 /* Allow a null timespec (wait forever). */ 2626 if (uap->uaddr2 == NULL) 2627 ts = NULL; 2628 else { 2629 error = copyin_timeout32(uap->uaddr2, &timeout); 2630 if (error != 0) 2631 return (error); 2632 if (timeout.tv_nsec >= 1000000000 || 2633 timeout.tv_nsec < 0) 2634 return (EINVAL); 2635 ts = &timeout; 2636 } 2637 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 2638 } 2639 2640 static _umtx_op_func op_table_compat32[] = { 2641 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ 2642 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ 2643 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ 2644 __umtx_op_wake, /* UMTX_OP_WAKE */ 2645 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ 2646 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ 2647 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 2648 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 2649 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/ 2650 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 2651 __umtx_op_cv_broadcast /* UMTX_OP_CV_BROADCAST */ 2652 }; 2653 2654 int 2655 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) 2656 { 2657 if ((unsigned)uap->op < UMTX_OP_MAX) 2658 return (*op_table_compat32[uap->op])(td, 2659 (struct _umtx_op_args *)uap); 2660 return (EINVAL); 2661 } 2662 #endif 2663 2664 void 2665 umtx_thread_init(struct thread *td) 2666 { 2667 td->td_umtxq = umtxq_alloc(); 2668 td->td_umtxq->uq_thread = td; 2669 } 2670 2671 void 2672 umtx_thread_fini(struct thread *td) 2673 { 2674 umtxq_free(td->td_umtxq); 2675 } 2676 2677 /* 2678 * It will be called when new thread is created, e.g fork(). 2679 */ 2680 void 2681 umtx_thread_alloc(struct thread *td) 2682 { 2683 struct umtx_q *uq; 2684 2685 uq = td->td_umtxq; 2686 uq->uq_inherited_pri = PRI_MAX; 2687 2688 KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); 2689 KASSERT(uq->uq_thread == td, ("uq_thread != td")); 2690 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); 2691 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); 2692 } 2693 2694 /* 2695 * exec() hook. 2696 */ 2697 static void 2698 umtx_exec_hook(void *arg __unused, struct proc *p __unused, 2699 struct image_params *imgp __unused) 2700 { 2701 umtx_thread_cleanup(curthread); 2702 } 2703 2704 /* 2705 * thread_exit() hook. 2706 */ 2707 void 2708 umtx_thread_exit(struct thread *td) 2709 { 2710 umtx_thread_cleanup(td); 2711 } 2712 2713 /* 2714 * clean up umtx data. 2715 */ 2716 static void 2717 umtx_thread_cleanup(struct thread *td) 2718 { 2719 struct umtx_q *uq; 2720 struct umtx_pi *pi; 2721 2722 if ((uq = td->td_umtxq) == NULL) 2723 return; 2724 2725 mtx_lock_spin(&umtx_lock); 2726 uq->uq_inherited_pri = PRI_MAX; 2727 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { 2728 pi->pi_owner = NULL; 2729 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); 2730 } 2731 thread_lock(td); 2732 td->td_flags &= ~TDF_UBORROWING; 2733 thread_unlock(td); 2734 mtx_unlock_spin(&umtx_lock); 2735 }
Cache object: 10dabe0a9e716b6f3a83c8918a35dba1
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]