FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_umtx.c
1 /*-
2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
11 * disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD: releng/6.4/sys/kern/kern_umtx.c 184239 2008-10-25 01:46:29Z alfred $");
30
31 #include "opt_compat.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/limits.h>
35 #include <sys/lock.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
38 #include <sys/proc.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysent.h>
41 #include <sys/systm.h>
42 #include <sys/sysproto.h>
43 #include <sys/eventhandler.h>
44 #include <sys/thr.h>
45 #include <sys/umtx.h>
46
47 #include <vm/vm.h>
48 #include <vm/vm_param.h>
49 #include <vm/pmap.h>
50 #include <vm/vm_map.h>
51 #include <vm/vm_object.h>
52
53 #ifdef COMPAT_IA32
54 #include <compat/freebsd32/freebsd32_proto.h>
55
56 #define UMTX_CONTESTED32 (-0x7fffffff - 1)
57 #endif
58
59 #define UMTX_PRIVATE 0
60 #define UMTX_SHARED 1
61
62 #define UMTX_STATIC_SHARED
63
64 struct umtx_key {
65 int type;
66 union {
67 struct {
68 vm_object_t object;
69 long offset;
70 } shared;
71 struct {
72 struct umtx *umtx;
73 long pid;
74 } private;
75 struct {
76 void *ptr;
77 long word;
78 } both;
79 } info;
80 };
81
82 struct umtx_q {
83 LIST_ENTRY(umtx_q) uq_next; /* Linked list for the hash. */
84 struct umtx_key uq_key; /* Umtx key. */
85 int uq_flags;
86 #define UQF_UMTXQ 0x0001
87 struct thread *uq_thread; /* The thread waits on. */
88 LIST_ENTRY(umtx_q) uq_rqnext; /* Linked list for requeuing. */
89 vm_offset_t uq_addr; /* Umtx's virtual address. */
90 };
91
92 LIST_HEAD(umtx_head, umtx_q);
93 struct umtxq_chain {
94 struct mtx uc_lock; /* Lock for this chain. */
95 struct umtx_head uc_queue; /* List of sleep queues. */
96 #define UCF_BUSY 0x01
97 #define UCF_WANT 0x02
98 int uc_flags;
99 };
100
101 #define GOLDEN_RATIO_PRIME 2654404609U
102 #define UMTX_CHAINS 128
103 #define UMTX_SHIFTS (__WORD_BIT - 7)
104
105 static struct umtxq_chain umtxq_chains[UMTX_CHAINS];
106 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
107
108 static void umtxq_init_chains(void *);
109 static int umtxq_hash(struct umtx_key *key);
110 static struct mtx *umtxq_mtx(int chain);
111 static void umtxq_lock(struct umtx_key *key);
112 static void umtxq_unlock(struct umtx_key *key);
113 static void umtxq_busy(struct umtx_key *key);
114 static void umtxq_unbusy(struct umtx_key *key);
115 static void umtxq_insert(struct umtx_q *uq);
116 static void umtxq_remove(struct umtx_q *uq);
117 static int umtxq_sleep(struct thread *td, struct umtx_key *key,
118 int prio, const char *wmesg, int timo);
119 static int umtxq_count(struct umtx_key *key);
120 static int umtxq_signal(struct umtx_key *key, int nr_wakeup);
121 #ifdef UMTX_DYNAMIC_SHARED
122 static void fork_handler(void *arg, struct proc *p1, struct proc *p2,
123 int flags);
124 #endif
125 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
126 static int umtx_key_get(struct thread *td, void *umtx,
127 struct umtx_key *key);
128 static void umtx_key_release(struct umtx_key *key);
129
130 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_init_chains, NULL);
131
132 struct umtx_q *
133 umtxq_alloc(void)
134 {
135 return (malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK));
136 }
137
138 void
139 umtxq_free(struct umtx_q *uq)
140 {
141 free(uq, M_UMTX);
142 }
143
144 static void
145 umtxq_init_chains(void *arg __unused)
146 {
147 int i;
148
149 for (i = 0; i < UMTX_CHAINS; ++i) {
150 mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL,
151 MTX_DEF | MTX_DUPOK);
152 LIST_INIT(&umtxq_chains[i].uc_queue);
153 umtxq_chains[i].uc_flags = 0;
154 }
155 #ifdef UMTX_DYNAMIC_SHARED
156 EVENTHANDLER_REGISTER(process_fork, fork_handler, 0, 10000);
157 #endif
158 }
159
160 static inline int
161 umtxq_hash(struct umtx_key *key)
162 {
163 unsigned n = (uintptr_t)key->info.both.ptr + key->info.both.word;
164 return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS);
165 }
166
167 static inline int
168 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
169 {
170 return (k1->type == k2->type &&
171 k1->info.both.ptr == k2->info.both.ptr &&
172 k1->info.both.word == k2->info.both.word);
173 }
174
175 static inline struct mtx *
176 umtxq_mtx(int chain)
177 {
178 return (&umtxq_chains[chain].uc_lock);
179 }
180
181 static inline void
182 umtxq_busy(struct umtx_key *key)
183 {
184 int chain = umtxq_hash(key);
185
186 mtx_assert(umtxq_mtx(chain), MA_OWNED);
187 while (umtxq_chains[chain].uc_flags & UCF_BUSY) {
188 umtxq_chains[chain].uc_flags |= UCF_WANT;
189 msleep(&umtxq_chains[chain], umtxq_mtx(chain),
190 0, "umtxq_busy", 0);
191 }
192 umtxq_chains[chain].uc_flags |= UCF_BUSY;
193 }
194
195 static inline void
196 umtxq_unbusy(struct umtx_key *key)
197 {
198 int chain = umtxq_hash(key);
199
200 mtx_assert(umtxq_mtx(chain), MA_OWNED);
201 KASSERT(umtxq_chains[chain].uc_flags & UCF_BUSY, ("not busy"));
202 umtxq_chains[chain].uc_flags &= ~UCF_BUSY;
203 if (umtxq_chains[chain].uc_flags & UCF_WANT) {
204 umtxq_chains[chain].uc_flags &= ~UCF_WANT;
205 wakeup(&umtxq_chains[chain]);
206 }
207 }
208
209 static inline void
210 umtxq_lock(struct umtx_key *key)
211 {
212 int chain = umtxq_hash(key);
213 mtx_lock(umtxq_mtx(chain));
214 }
215
216 static inline void
217 umtxq_unlock(struct umtx_key *key)
218 {
219 int chain = umtxq_hash(key);
220 mtx_unlock(umtxq_mtx(chain));
221 }
222
223 /*
224 * Insert a thread onto the umtx queue.
225 */
226 static inline void
227 umtxq_insert(struct umtx_q *uq)
228 {
229 struct umtx_head *head;
230 int chain = umtxq_hash(&uq->uq_key);
231
232 mtx_assert(umtxq_mtx(chain), MA_OWNED);
233 head = &umtxq_chains[chain].uc_queue;
234 LIST_INSERT_HEAD(head, uq, uq_next);
235 uq->uq_flags |= UQF_UMTXQ;
236 }
237
238 /*
239 * Remove thread from the umtx queue.
240 */
241 static inline void
242 umtxq_remove(struct umtx_q *uq)
243 {
244 mtx_assert(umtxq_mtx(umtxq_hash(&uq->uq_key)), MA_OWNED);
245 if (uq->uq_flags & UQF_UMTXQ) {
246 LIST_REMOVE(uq, uq_next);
247 /* turning off UQF_UMTXQ should be the last thing. */
248 uq->uq_flags &= ~UQF_UMTXQ;
249 }
250 }
251
252 static int
253 umtxq_count(struct umtx_key *key)
254 {
255 struct umtx_q *uq;
256 struct umtx_head *head;
257 int chain, count = 0;
258
259 chain = umtxq_hash(key);
260 mtx_assert(umtxq_mtx(chain), MA_OWNED);
261 head = &umtxq_chains[chain].uc_queue;
262 LIST_FOREACH(uq, head, uq_next) {
263 if (umtx_key_match(&uq->uq_key, key)) {
264 if (++count > 1)
265 break;
266 }
267 }
268 return (count);
269 }
270
271 static int
272 umtxq_signal(struct umtx_key *key, int n_wake)
273 {
274 struct umtx_q *uq, *next;
275 struct umtx_head *head;
276 struct thread *blocked = NULL;
277 int chain, ret;
278
279 ret = 0;
280 chain = umtxq_hash(key);
281 mtx_assert(umtxq_mtx(chain), MA_OWNED);
282 head = &umtxq_chains[chain].uc_queue;
283 for (uq = LIST_FIRST(head); uq; uq = next) {
284 next = LIST_NEXT(uq, uq_next);
285 if (umtx_key_match(&uq->uq_key, key)) {
286 blocked = uq->uq_thread;
287 umtxq_remove(uq);
288 wakeup(blocked);
289 if (++ret >= n_wake)
290 break;
291 }
292 }
293 return (ret);
294 }
295
296 static inline int
297 umtxq_sleep(struct thread *td, struct umtx_key *key, int priority,
298 const char *wmesg, int timo)
299 {
300 int chain = umtxq_hash(key);
301 int error = msleep(td, umtxq_mtx(chain), priority, wmesg, timo);
302 if (error == EWOULDBLOCK)
303 error = ETIMEDOUT;
304 return (error);
305 }
306
307 static int
308 umtx_key_get(struct thread *td, void *umtx, struct umtx_key *key)
309 {
310 #if defined(UMTX_STATIC_SHARED)
311 vm_map_t map;
312 vm_map_entry_t entry;
313 vm_pindex_t pindex;
314 vm_prot_t prot;
315 boolean_t wired;
316
317 map = &td->td_proc->p_vmspace->vm_map;
318 if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE,
319 &entry, &key->info.shared.object, &pindex, &prot,
320 &wired) != KERN_SUCCESS) {
321 return EFAULT;
322 }
323
324 if (VM_INHERIT_SHARE == entry->inheritance) {
325 key->type = UMTX_SHARED;
326 key->info.shared.offset = entry->offset + entry->start -
327 (vm_offset_t)umtx;
328 vm_object_reference(key->info.shared.object);
329 } else {
330 key->type = UMTX_PRIVATE;
331 key->info.private.umtx = umtx;
332 key->info.private.pid = td->td_proc->p_pid;
333 }
334 vm_map_lookup_done(map, entry);
335 #else
336 key->type = UMTX_PRIVATE;
337 key->info.private.umtx = umtx;
338 key->info.private.pid = td->td_proc->p_pid;
339 #endif
340 return (0);
341 }
342
343 static inline void
344 umtx_key_release(struct umtx_key *key)
345 {
346 if (key->type == UMTX_SHARED)
347 vm_object_deallocate(key->info.shared.object);
348 }
349
350 static inline int
351 umtxq_queue_me(struct thread *td, void *umtx, struct umtx_q *uq)
352 {
353 int error;
354
355 if ((error = umtx_key_get(td, umtx, &uq->uq_key)) != 0)
356 return (error);
357
358 uq->uq_addr = (vm_offset_t)umtx;
359 uq->uq_thread = td;
360 umtxq_lock(&uq->uq_key);
361 /* hmm, for condition variable, we don't need busy flag. */
362 umtxq_busy(&uq->uq_key);
363 umtxq_insert(uq);
364 umtxq_unbusy(&uq->uq_key);
365 umtxq_unlock(&uq->uq_key);
366 return (0);
367 }
368
369 static int
370 _do_lock(struct thread *td, struct umtx *umtx, long id, int timo)
371 {
372 struct umtx_q *uq;
373 intptr_t owner;
374 intptr_t old;
375 int error = 0;
376
377 uq = td->td_umtxq;
378 /*
379 * Care must be exercised when dealing with umtx structure. It
380 * can fault on any access.
381 */
382
383 for (;;) {
384 /*
385 * Try the uncontested case. This should be done in userland.
386 */
387 owner = casuptr((intptr_t *)&umtx->u_owner,
388 UMTX_UNOWNED, id);
389
390 /* The acquire succeeded. */
391 if (owner == UMTX_UNOWNED)
392 return (0);
393
394 /* The address was invalid. */
395 if (owner == -1)
396 return (EFAULT);
397
398 /* If no one owns it but it is contested try to acquire it. */
399 if (owner == UMTX_CONTESTED) {
400 owner = casuptr((intptr_t *)&umtx->u_owner,
401 UMTX_CONTESTED, id | UMTX_CONTESTED);
402
403 if (owner == UMTX_CONTESTED)
404 return (0);
405
406 /* The address was invalid. */
407 if (owner == -1)
408 return (EFAULT);
409
410 /* If this failed the lock has changed, restart. */
411 continue;
412 }
413
414 /*
415 * If we caught a signal, we have retried and now
416 * exit immediately.
417 */
418 if (error || (error = umtxq_queue_me(td, umtx, uq)) != 0)
419 return (error);
420
421 /*
422 * Set the contested bit so that a release in user space
423 * knows to use the system call for unlock. If this fails
424 * either some one else has acquired the lock or it has been
425 * released.
426 */
427 old = casuptr((intptr_t *)&umtx->u_owner, owner,
428 owner | UMTX_CONTESTED);
429
430 /* The address was invalid. */
431 if (old == -1) {
432 umtxq_lock(&uq->uq_key);
433 umtxq_busy(&uq->uq_key);
434 umtxq_remove(uq);
435 umtxq_unbusy(&uq->uq_key);
436 umtxq_unlock(&uq->uq_key);
437 umtx_key_release(&uq->uq_key);
438 return (EFAULT);
439 }
440
441 /*
442 * We set the contested bit, sleep. Otherwise the lock changed
443 * and we need to retry or we lost a race to the thread
444 * unlocking the umtx.
445 */
446 umtxq_lock(&uq->uq_key);
447 if (old == owner && (uq->uq_flags & UQF_UMTXQ)) {
448 error = umtxq_sleep(td, &uq->uq_key, PCATCH,
449 "umtx", timo);
450 }
451 umtxq_busy(&uq->uq_key);
452 umtxq_remove(uq);
453 umtxq_unbusy(&uq->uq_key);
454 umtxq_unlock(&uq->uq_key);
455 umtx_key_release(&uq->uq_key);
456 }
457
458 return (0);
459 }
460
461 static int
462 do_lock(struct thread *td, struct umtx *umtx, long id,
463 struct timespec *timeout)
464 {
465 struct timespec ts, ts2, ts3;
466 struct timeval tv;
467 int error;
468
469 if (timeout == NULL) {
470 error = _do_lock(td, umtx, id, 0);
471 } else {
472 getnanouptime(&ts);
473 timespecadd(&ts, timeout);
474 TIMESPEC_TO_TIMEVAL(&tv, timeout);
475 for (;;) {
476 error = _do_lock(td, umtx, id, tvtohz(&tv));
477 if (error != ETIMEDOUT)
478 break;
479 getnanouptime(&ts2);
480 if (timespeccmp(&ts2, &ts, >=)) {
481 error = ETIMEDOUT;
482 break;
483 }
484 ts3 = ts;
485 timespecsub(&ts3, &ts2);
486 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
487 }
488 }
489 /*
490 * This lets userland back off critical region if needed.
491 */
492 if (error == ERESTART)
493 error = EINTR;
494 return (error);
495 }
496
497 static int
498 do_unlock(struct thread *td, struct umtx *umtx, long id)
499 {
500 struct umtx_key key;
501 intptr_t owner;
502 intptr_t old;
503 int error;
504 int count;
505
506 /*
507 * Make sure we own this mtx.
508 *
509 * XXX Need a {fu,su}ptr this is not correct on arch where
510 * sizeof(intptr_t) != sizeof(long).
511 */
512 if ((owner = fuword(&umtx->u_owner)) == -1)
513 return (EFAULT);
514
515 if ((owner & ~UMTX_CONTESTED) != id)
516 return (EPERM);
517
518 /* We should only ever be in here for contested locks */
519 if ((owner & UMTX_CONTESTED) == 0)
520 return (EINVAL);
521
522 if ((error = umtx_key_get(td, umtx, &key)) != 0)
523 return (error);
524
525 umtxq_lock(&key);
526 umtxq_busy(&key);
527 count = umtxq_count(&key);
528 umtxq_unlock(&key);
529
530 /*
531 * When unlocking the umtx, it must be marked as unowned if
532 * there is zero or one thread only waiting for it.
533 * Otherwise, it must be marked as contested.
534 */
535 old = casuptr((intptr_t *)&umtx->u_owner, owner,
536 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
537 umtxq_lock(&key);
538 umtxq_signal(&key, 0);
539 umtxq_unbusy(&key);
540 umtxq_unlock(&key);
541 umtx_key_release(&key);
542 if (old == -1)
543 return (EFAULT);
544 if (old != owner)
545 return (EINVAL);
546 return (0);
547 }
548
549 #ifdef COMPAT_IA32
550 static int
551 _do_lock32(struct thread *td, uint32_t *m, uint32_t id, int timo)
552 {
553 struct umtx_q *uq;
554 int32_t owner;
555 int32_t old;
556 int error = 0;
557
558 uq = td->td_umtxq;
559 /*
560 * Care must be exercised when dealing with umtx structure. It
561 * can fault on any access.
562 */
563
564 for (;;) {
565 /*
566 * Try the uncontested case. This should be done in userland.
567 */
568 owner = casuword32(m, UMTX_UNOWNED, id);
569
570 /* The acquire succeeded. */
571 if (owner == UMTX_UNOWNED)
572 return (0);
573
574 /* The address was invalid. */
575 if (owner == -1)
576 return (EFAULT);
577
578 /* If no one owns it but it is contested try to acquire it. */
579 if (owner == UMTX_CONTESTED32) {
580 owner = casuword32(m,
581 UMTX_CONTESTED32, id | UMTX_CONTESTED32);
582
583 if (owner == UMTX_CONTESTED32)
584 return (0);
585
586 /* The address was invalid. */
587 if (owner == -1)
588 return (EFAULT);
589
590 /* If this failed the lock has changed, restart. */
591 continue;
592 }
593
594 /*
595 * If we caught a signal, we have retried and now
596 * exit immediately.
597 */
598 if (error || (error = umtxq_queue_me(td, m, uq)) != 0)
599 return (error);
600
601 /*
602 * Set the contested bit so that a release in user space
603 * knows to use the system call for unlock. If this fails
604 * either some one else has acquired the lock or it has been
605 * released.
606 */
607 old = casuword32(m, owner, owner | UMTX_CONTESTED32);
608
609 /* The address was invalid. */
610 if (old == -1) {
611 umtxq_lock(&uq->uq_key);
612 umtxq_busy(&uq->uq_key);
613 umtxq_remove(uq);
614 umtxq_unbusy(&uq->uq_key);
615 umtxq_unlock(&uq->uq_key);
616 umtx_key_release(&uq->uq_key);
617 return (EFAULT);
618 }
619
620 /*
621 * We set the contested bit, sleep. Otherwise the lock changed
622 * and we need to retry or we lost a race to the thread
623 * unlocking the umtx.
624 */
625 umtxq_lock(&uq->uq_key);
626 if (old == owner && (uq->uq_flags & UQF_UMTXQ)) {
627 error = umtxq_sleep(td, &uq->uq_key, PCATCH,
628 "umtx", timo);
629 }
630 umtxq_busy(&uq->uq_key);
631 umtxq_remove(uq);
632 umtxq_unbusy(&uq->uq_key);
633 umtxq_unlock(&uq->uq_key);
634 umtx_key_release(&uq->uq_key);
635 }
636
637 return (0);
638 }
639
640 static int
641 do_lock32(struct thread *td, void *m, uint32_t id,
642 struct timespec *timeout)
643 {
644 struct timespec ts, ts2, ts3;
645 struct timeval tv;
646 int error;
647
648 if (timeout == NULL) {
649 error = _do_lock32(td, m, id, 0);
650 } else {
651 getnanouptime(&ts);
652 timespecadd(&ts, timeout);
653 TIMESPEC_TO_TIMEVAL(&tv, timeout);
654 for (;;) {
655 error = _do_lock32(td, m, id, tvtohz(&tv));
656 if (error != ETIMEDOUT)
657 break;
658 getnanouptime(&ts2);
659 if (timespeccmp(&ts2, &ts, >=)) {
660 error = ETIMEDOUT;
661 break;
662 }
663 ts3 = ts;
664 timespecsub(&ts3, &ts2);
665 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
666 }
667 }
668 /*
669 * This lets userland back off critical region if needed.
670 */
671 if (error == ERESTART)
672 error = EINTR;
673 return (error);
674 }
675
676 static int
677 do_unlock32(struct thread *td, uint32_t *m, uint32_t id)
678 {
679 struct umtx_key key;
680 int32_t owner;
681 int32_t old;
682 int error;
683 int count;
684
685 /*
686 * Make sure we own this mtx.
687 *
688 * XXX Need a {fu,su}ptr this is not correct on arch where
689 * sizeof(intptr_t) != sizeof(long).
690 */
691 if ((owner = fuword32(m)) == -1)
692 return (EFAULT);
693
694 if ((owner & ~UMTX_CONTESTED32) != id)
695 return (EPERM);
696
697 /* We should only ever be in here for contested locks */
698 if ((owner & UMTX_CONTESTED32) == 0)
699 return (EINVAL);
700
701 if ((error = umtx_key_get(td, m, &key)) != 0)
702 return (error);
703
704 umtxq_lock(&key);
705 umtxq_busy(&key);
706 count = umtxq_count(&key);
707 umtxq_unlock(&key);
708
709 /*
710 * When unlocking the umtx, it must be marked as unowned if
711 * there is zero or one thread only waiting for it.
712 * Otherwise, it must be marked as contested.
713 */
714 old = casuword32(m, owner,
715 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED32);
716 umtxq_lock(&key);
717 umtxq_signal(&key, 0);
718 umtxq_unbusy(&key);
719 umtxq_unlock(&key);
720 umtx_key_release(&key);
721 if (old == -1)
722 return (EFAULT);
723 if (old != owner)
724 return (EINVAL);
725 return (0);
726 }
727 #endif
728
729 static int
730 do_wait(struct thread *td, struct umtx *umtx, long id, struct timespec *timeout,
731 int compat32)
732 {
733 struct umtx_q *uq;
734 struct timespec ts, ts2, ts3;
735 struct timeval tv;
736 long tmp;
737 int error = 0;
738
739 uq = td->td_umtxq;
740 if ((error = umtxq_queue_me(td, umtx, uq)) != 0)
741 return (error);
742 if (compat32 == 0)
743 tmp = fuword(&umtx->u_owner);
744 else
745 tmp = fuword32(&umtx->u_owner);
746 umtxq_lock(&uq->uq_key);
747 if (tmp != id) {
748 umtxq_remove(uq);
749 } else if (timeout == NULL) {
750 if (uq->uq_flags & UQF_UMTXQ)
751 error = umtxq_sleep(td, &uq->uq_key,
752 PCATCH, "ucond", 0);
753 } else {
754 getnanouptime(&ts);
755 timespecadd(&ts, timeout);
756 TIMESPEC_TO_TIMEVAL(&tv, timeout);
757 for (;;) {
758 if (uq->uq_flags & UQF_UMTXQ) {
759 error = umtxq_sleep(td, &uq->uq_key, PCATCH,
760 "ucondt", tvtohz(&tv));
761 }
762 if (error != ETIMEDOUT)
763 break;
764 getnanouptime(&ts2);
765 if (timespeccmp(&ts2, &ts, >=)) {
766 error = ETIMEDOUT;
767 break;
768 }
769 ts3 = ts;
770 timespecsub(&ts3, &ts2);
771 TIMESPEC_TO_TIMEVAL(&tv, &ts3);
772 }
773 }
774 if (error != 0) {
775 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
776 /*
777 * If we concurrently got do_cv_signal()d
778 * and we got an error or UNIX signals or a timeout,
779 * then, perform another umtxq_signal to avoid
780 * consuming the wakeup. This may cause supurious
781 * wakeup for another thread which was just queued,
782 * but SUSV3 explicitly allows supurious wakeup to
783 * occur, and indeed a kernel based implementation
784 * can not avoid it.
785 */
786 if (!umtxq_signal(&uq->uq_key, 1))
787 error = 0;
788 }
789 if (error == ERESTART)
790 error = EINTR;
791 }
792 umtxq_remove(uq);
793 umtxq_unlock(&uq->uq_key);
794 umtx_key_release(&uq->uq_key);
795 return (error);
796 }
797
798 int
799 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake)
800 {
801 struct umtx_key key;
802 int ret;
803
804 if ((ret = umtx_key_get(td, uaddr, &key)) != 0)
805 return (ret);
806 umtxq_lock(&key);
807 ret = umtxq_signal(&key, n_wake);
808 umtxq_unlock(&key);
809 umtx_key_release(&key);
810 return (0);
811 }
812
813 int
814 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
815 /* struct umtx *umtx */
816 {
817 return _do_lock(td, uap->umtx, td->td_tid, 0);
818 }
819
820 int
821 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
822 /* struct umtx *umtx */
823 {
824 return do_unlock(td, uap->umtx, td->td_tid);
825 }
826
827 int
828 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
829 {
830 struct timespec timeout;
831 struct timespec *ts;
832 int error;
833
834 switch(uap->op) {
835 case UMTX_OP_LOCK:
836 /* Allow a null timespec (wait forever). */
837 if (uap->uaddr2 == NULL)
838 ts = NULL;
839 else {
840 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
841 if (error != 0)
842 break;
843 if (timeout.tv_nsec >= 1000000000 ||
844 timeout.tv_nsec < 0) {
845 error = EINVAL;
846 break;
847 }
848 ts = &timeout;
849 }
850 error = do_lock(td, uap->umtx, uap->id, ts);
851 break;
852 case UMTX_OP_UNLOCK:
853 error = do_unlock(td, uap->umtx, uap->id);
854 break;
855 case UMTX_OP_WAIT:
856 /* Allow a null timespec (wait forever). */
857 if (uap->uaddr2 == NULL)
858 ts = NULL;
859 else {
860 error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
861 if (error != 0)
862 break;
863 if (timeout.tv_nsec >= 1000000000 ||
864 timeout.tv_nsec < 0) {
865 error = EINVAL;
866 break;
867 }
868 ts = &timeout;
869 }
870 error = do_wait(td, uap->umtx, uap->id, ts, 0);
871 break;
872 case UMTX_OP_WAKE:
873 error = kern_umtx_wake(td, uap->umtx, uap->id);
874 break;
875 default:
876 error = EINVAL;
877 break;
878 }
879 return (error);
880 }
881
882 #ifdef COMPAT_IA32
883 int
884 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
885 /* struct umtx *umtx */
886 {
887 return (do_lock32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
888 }
889
890 int
891 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
892 /* struct umtx *umtx */
893 {
894 return (do_unlock32(td, (uint32_t *)uap->umtx, td->td_tid));
895 }
896
897 struct timespec32 {
898 u_int32_t tv_sec;
899 u_int32_t tv_nsec;
900 };
901
902 static inline int
903 copyin_timeout32(void *addr, struct timespec *tsp)
904 {
905 struct timespec32 ts32;
906 int error;
907
908 error = copyin(addr, &ts32, sizeof(struct timespec32));
909 if (error == 0) {
910 tsp->tv_sec = ts32.tv_sec;
911 tsp->tv_nsec = ts32.tv_nsec;
912 }
913 return (error);
914 }
915
916 static int
917 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
918 {
919 struct timespec *ts, timeout;
920 int error;
921
922 /* Allow a null timespec (wait forever). */
923 if (uap->uaddr2 == NULL)
924 ts = NULL;
925 else {
926 error = copyin_timeout32(uap->uaddr2, &timeout);
927 if (error != 0)
928 return (error);
929 if (timeout.tv_nsec >= 1000000000 ||
930 timeout.tv_nsec < 0) {
931 return (EINVAL);
932 }
933 ts = &timeout;
934 }
935 return (do_lock32(td, uap->umtx, uap->id, ts));
936 }
937
938 static int
939 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
940 {
941 return (do_unlock32(td, (uint32_t *)uap->umtx, (uint32_t)uap->id));
942 }
943
944 static int
945 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
946 {
947 struct timespec *ts, timeout;
948 int error;
949
950 if (uap->uaddr2 == NULL)
951 ts = NULL;
952 else {
953 error = copyin_timeout32(uap->uaddr2, &timeout);
954 if (error != 0)
955 return (error);
956 if (timeout.tv_nsec >= 1000000000 ||
957 timeout.tv_nsec < 0)
958 return (EINVAL);
959 ts = &timeout;
960 }
961 return do_wait(td, uap->umtx, uap->id, ts, 1);
962 }
963
964 int
965 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
966 {
967
968 switch ((unsigned)uap->op) {
969 case UMTX_OP_LOCK:
970 return __umtx_op_lock_umtx_compat32(td,
971 (struct _umtx_op_args *)uap);
972 case UMTX_OP_UNLOCK:
973 return __umtx_op_unlock_umtx_compat32(td,
974 (struct _umtx_op_args *)uap);
975 case UMTX_OP_WAIT:
976 return __umtx_op_wait_compat32(td,
977 (struct _umtx_op_args *)uap);
978 case UMTX_OP_WAKE:
979 return kern_umtx_wake(td, (uint32_t *)uap->umtx, uap->id);
980 default:
981 return (EINVAL);
982 }
983 }
984 #endif
Cache object: 779e955a93f81dbcee0bb774a1b65c31
|