kern_umtx.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
    3  * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice unmodified, this list of conditions, and the following
   11  *    disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/6.4/sys/kern/kern_umtx.c 184239 2008-10-25 01:46:29Z alfred $");
   30 
   31 #include "opt_compat.h"
   32 #include <sys/param.h>
   33 #include <sys/kernel.h>
   34 #include <sys/limits.h>
   35 #include <sys/lock.h>
   36 #include <sys/malloc.h>
   37 #include <sys/mutex.h>
   38 #include <sys/proc.h>
   39 #include <sys/sysctl.h>
   40 #include <sys/sysent.h>
   41 #include <sys/systm.h>
   42 #include <sys/sysproto.h>
   43 #include <sys/eventhandler.h>
   44 #include <sys/thr.h>
   45 #include <sys/umtx.h>
   46 
   47 #include <vm/vm.h>
   48 #include <vm/vm_param.h>
   49 #include <vm/pmap.h>
   50 #include <vm/vm_map.h>
   51 #include <vm/vm_object.h>
   52 
   53 #ifdef COMPAT_IA32
   54 #include <compat/freebsd32/freebsd32_proto.h>
   55 
   56 #define UMTX_CONTESTED32 (-0x7fffffff - 1)
   57 #endif
   58 
   59 #define UMTX_PRIVATE    0
   60 #define UMTX_SHARED     1
   61 
   62 #define UMTX_STATIC_SHARED
   63 
   64 struct umtx_key {
   65         int     type;
   66         union {
   67                 struct {
   68                         vm_object_t     object;
   69                         long            offset;
   70                 } shared;
   71                 struct {
   72                         struct umtx     *umtx;
   73                         long            pid;
   74                 } private;
   75                 struct {
   76                         void            *ptr;
   77                         long            word;
   78                 } both;
   79         } info;
   80 };
   81 
   82 struct umtx_q {
   83         LIST_ENTRY(umtx_q)      uq_next;        /* Linked list for the hash. */
   84         struct umtx_key         uq_key;         /* Umtx key. */
   85         int                     uq_flags;
   86 #define UQF_UMTXQ       0x0001
   87         struct thread           *uq_thread;     /* The thread waits on. */
   88         LIST_ENTRY(umtx_q)      uq_rqnext;      /* Linked list for requeuing. */
   89         vm_offset_t             uq_addr;        /* Umtx's virtual address. */
   90 };
   91 
   92 LIST_HEAD(umtx_head, umtx_q);
   93 struct umtxq_chain {
   94         struct mtx              uc_lock;        /* Lock for this chain. */
   95         struct umtx_head        uc_queue;       /* List of sleep queues. */
   96 #define UCF_BUSY                0x01
   97 #define UCF_WANT                0x02
   98         int                     uc_flags;
   99 };
  100 
  101 #define GOLDEN_RATIO_PRIME      2654404609U
  102 #define UMTX_CHAINS             128
  103 #define UMTX_SHIFTS             (__WORD_BIT - 7)
  104 
  105 static struct umtxq_chain umtxq_chains[UMTX_CHAINS];
  106 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
  107 
  108 static void umtxq_init_chains(void *);
  109 static int umtxq_hash(struct umtx_key *key);
  110 static struct mtx *umtxq_mtx(int chain);
  111 static void umtxq_lock(struct umtx_key *key);
  112 static void umtxq_unlock(struct umtx_key *key);
  113 static void umtxq_busy(struct umtx_key *key);
  114 static void umtxq_unbusy(struct umtx_key *key);
  115 static void umtxq_insert(struct umtx_q *uq);
  116 static void umtxq_remove(struct umtx_q *uq);
  117 static int umtxq_sleep(struct thread *td, struct umtx_key *key,
  118         int prio, const char *wmesg, int timo);
  119 static int umtxq_count(struct umtx_key *key);
  120 static int umtxq_signal(struct umtx_key *key, int nr_wakeup);
  121 #ifdef UMTX_DYNAMIC_SHARED
  122 static void fork_handler(void *arg, struct proc *p1, struct proc *p2,
  123         int flags);
  124 #endif
  125 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
  126 static int umtx_key_get(struct thread *td, void *umtx,
  127         struct umtx_key *key);
  128 static void umtx_key_release(struct umtx_key *key);
  129 
  130 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_init_chains, NULL);
  131 
  132 struct umtx_q *
  133 umtxq_alloc(void)
  134 {
  135         return (malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK));
  136 }
  137 
  138 void
  139 umtxq_free(struct umtx_q *uq)
  140 {
  141         free(uq, M_UMTX);
  142 }
  143 
  144 static void
  145 umtxq_init_chains(void *arg __unused)
  146 {
  147         int i;
  148 
  149         for (i = 0; i < UMTX_CHAINS; ++i) {
  150                 mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL,
  151                          MTX_DEF | MTX_DUPOK);
  152                 LIST_INIT(&umtxq_chains[i].uc_queue);
  153                 umtxq_chains[i].uc_flags = 0;
  154         }
  155 #ifdef UMTX_DYNAMIC_SHARED
  156         EVENTHANDLER_REGISTER(process_fork, fork_handler, 0, 10000);
  157 #endif
  158 }
  159 
  160 static inline int
  161 umtxq_hash(struct umtx_key *key)
  162 {
  163         unsigned n = (uintptr_t)key->info.both.ptr + key->info.both.word;
  164         return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS);
  165 }
  166 
  167 static inline int
  168 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
  169 {
  170         return (k1->type == k2->type &&
  171                 k1->info.both.ptr == k2->info.both.ptr &&
  172                 k1->info.both.word == k2->info.both.word);
  173 }
  174 
  175 static inline struct mtx *
  176 umtxq_mtx(int chain)
  177 {
  178         return (&umtxq_chains[chain].uc_lock);
  179 }
  180 
  181 static inline void
  182 umtxq_busy(struct umtx_key *key)
  183 {
  184         int chain = umtxq_hash(key);
  185 
  186         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  187         while (umtxq_chains[chain].uc_flags & UCF_BUSY) {
  188                 umtxq_chains[chain].uc_flags |= UCF_WANT;
  189                 msleep(&umtxq_chains[chain], umtxq_mtx(chain),
  190                     0, "umtxq_busy", 0);
  191         }
  192         umtxq_chains[chain].uc_flags |= UCF_BUSY;
  193 }
  194 
  195 static inline void
  196 umtxq_unbusy(struct umtx_key *key)
  197 {
  198         int chain = umtxq_hash(key);
  199 
  200         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  201         KASSERT(umtxq_chains[chain].uc_flags & UCF_BUSY, ("not busy"));
  202         umtxq_chains[chain].uc_flags &= ~UCF_BUSY;
  203         if (umtxq_chains[chain].uc_flags & UCF_WANT) {
  204                 umtxq_chains[chain].uc_flags &= ~UCF_WANT;
  205                 wakeup(&umtxq_chains[chain]);
  206         }
  207 }
  208 
  209 static inline void
  210 umtxq_lock(struct umtx_key *key)
  211 {
  212         int chain = umtxq_hash(key);
  213         mtx_lock(umtxq_mtx(chain));
  214 }
  215 
  216 static inline void
  217 umtxq_unlock(struct umtx_key *key)
  218 {
  219         int chain = umtxq_hash(key);
  220         mtx_unlock(umtxq_mtx(chain));
  221 }
  222 
  223 /*
  224  * Insert a thread onto the umtx queue.
  225  */
  226 static inline void
  227 umtxq_insert(struct umtx_q *uq)
  228 {
  229         struct umtx_head *head;
  230         int chain = umtxq_hash(&uq->uq_key);
  231 
  232         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  233         head = &umtxq_chains[chain].uc_queue;
  234         LIST_INSERT_HEAD(head, uq, uq_next);
  235         uq->uq_flags |= UQF_UMTXQ;
  236 }
  237 
  238 /*
  239  * Remove thread from the umtx queue.
  240  */
  241 static inline void
  242 umtxq_remove(struct umtx_q *uq)
  243 {
  244         mtx_assert(umtxq_mtx(umtxq_hash(&uq->uq_key)), MA_OWNED);
  245         if (uq->uq_flags & UQF_UMTXQ) {
  246                 LIST_REMOVE(uq, uq_next);
  247                 /* turning off UQF_UMTXQ should be the last thing. */
  248                 uq->uq_flags &= ~UQF_UMTXQ;
  249         }
  250 }
  251 
  252 static int
  253 umtxq_count(struct umtx_key *key)
  254 {
  255         struct umtx_q *uq;
  256         struct umtx_head *head;
  257         int chain, count = 0;
  258 
  259         chain = umtxq_hash(key);
  260         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  261         head = &umtxq_chains[chain].uc_queue;
  262         LIST_FOREACH(uq, head, uq_next) {
  263                 if (umtx_key_match(&uq->uq_key, key)) {
  264                         if (++count > 1)
  265                                 break;
  266                 }
  267         }
  268         return (count);
  269 }
  270 
  271 static int
  272 umtxq_signal(struct umtx_key *key, int n_wake)
  273 {
  274         struct umtx_q *uq, *next;
  275         struct umtx_head *head;
  276         struct thread *blocked = NULL;
  277         int chain, ret;
  278 
  279         ret = 0;
  280         chain = umtxq_hash(key);
  281         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  282         head = &umtxq_chains[chain].uc_queue;
  283         for (uq = LIST_FIRST(head); uq; uq = next) {
  284                 next = LIST_NEXT(uq, uq_next);
  285                 if (umtx_key_match(&uq->uq_key, key)) {
  286                         blocked = uq->uq_thread;
  287                         umtxq_remove(uq);
  288                         wakeup(blocked);
  289                         if (++ret >= n_wake)
  290                                 break;
  291                 }
  292         }
  293         return (ret);
  294 }
  295 
  296 static inline int
  297 umtxq_sleep(struct thread *td, struct umtx_key *key, int priority,
  298             const char *wmesg, int timo)
  299 {
  300         int chain = umtxq_hash(key);
  301         int error = msleep(td, umtxq_mtx(chain), priority, wmesg, timo);
  302         if (error == EWOULDBLOCK)
  303                 error = ETIMEDOUT;
  304         return (error);
  305 }
  306 
  307 static int
  308 umtx_key_get(struct thread *td, void *umtx, struct umtx_key *key)
  309 {
  310 #if defined(UMTX_STATIC_SHARED)
  311         vm_map_t map;
  312         vm_map_entry_t entry;
  313         vm_pindex_t pindex;
  314         vm_prot_t prot;
  315         boolean_t wired;
  316 
  317         map = &td->td_proc->p_vmspace->vm_map;
  318         if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE,
  319             &entry, &key->info.shared.object, &pindex, &prot,
  320             &wired) != KERN_SUCCESS) {
  321                 return EFAULT;
  322         }
  323 
  324         if (VM_INHERIT_SHARE == entry->inheritance) {
  325                 key->type = UMTX_SHARED;
  326                 key->info.shared.offset = entry->offset + entry->start -
  327                         (vm_offset_t)umtx;
  328                 vm_object_reference(key->info.shared.object);
  329         } else {
  330                 key->type = UMTX_PRIVATE;
  331                 key->info.private.umtx = umtx;
  332                 key->info.private.pid  = td->td_proc->p_pid;
  333         }
  334         vm_map_lookup_done(map, entry);
  335 #else
  336         key->type = UMTX_PRIVATE;
  337         key->info.private.umtx = umtx;
  338         key->info.private.pid  = td->td_proc->p_pid;
  339 #endif
  340         return (0);
  341 }
  342 
  343 static inline void
  344 umtx_key_release(struct umtx_key *key)
  345 {
  346         if (key->type == UMTX_SHARED)
  347                 vm_object_deallocate(key->info.shared.object);
  348 }
  349 
  350 static inline int
  351 umtxq_queue_me(struct thread *td, void *umtx, struct umtx_q *uq)
  352 {
  353         int error;
  354 
  355         if ((error = umtx_key_get(td, umtx, &uq->uq_key)) != 0)
  356                 return (error);
  357 
  358         uq->uq_addr = (vm_offset_t)umtx;
  359         uq->uq_thread = td;
  360         umtxq_lock(&uq->uq_key);
  361         /* hmm, for condition variable, we don't need busy flag. */
  362         umtxq_busy(&uq->uq_key);
  363         umtxq_insert(uq);
  364         umtxq_unbusy(&uq->uq_key);
  365         umtxq_unlock(&uq->uq_key);
  366         return (0);
  367 }
  368 
  369 static int
  370 _do_lock(struct thread *td, struct umtx *umtx, long id, int timo)
  371 {
  372         struct umtx_q *uq;
  373         intptr_t owner;
  374         intptr_t old;
  375         int error = 0;
  376 
  377         uq = td->td_umtxq;
  378         /*
  379          * Care must be exercised when dealing with umtx structure.  It
  380          * can fault on any access.
  381          */
  382 
  383         for (;;) {
  384                 /*
  385                  * Try the uncontested case.  This should be done in userland.
  386                  */
  387                 owner = casuptr((intptr_t *)&umtx->u_owner,
  388                     UMTX_UNOWNED, id);
  389 
  390                 /* The acquire succeeded. */
  391                 if (owner == UMTX_UNOWNED)
  392                         return (0);
  393 
  394                 /* The address was invalid. */
  395                 if (owner == -1)
  396                         return (EFAULT);
  397 
  398                 /* If no one owns it but it is contested try to acquire it. */
  399                 if (owner == UMTX_CONTESTED) {
  400                         owner = casuptr((intptr_t *)&umtx->u_owner,
  401                             UMTX_CONTESTED, id | UMTX_CONTESTED);
  402 
  403                         if (owner == UMTX_CONTESTED)
  404                                 return (0);
  405 
  406                         /* The address was invalid. */
  407                         if (owner == -1)
  408                                 return (EFAULT);
  409 
  410                         /* If this failed the lock has changed, restart. */
  411                         continue;
  412                 }
  413 
  414                 /*
  415                  * If we caught a signal, we have retried and now
  416                  * exit immediately.
  417                  */
  418                 if (error || (error = umtxq_queue_me(td, umtx, uq)) != 0)
  419                         return (error);
  420 
  421                 /*
  422                  * Set the contested bit so that a release in user space
  423                  * knows to use the system call for unlock.  If this fails
  424                  * either some one else has acquired the lock or it has been
  425                  * released.
  426                  */
  427                 old = casuptr((intptr_t *)&umtx->u_owner, owner,
  428                     owner | UMTX_CONTESTED);
  429 
  430                 /* The address was invalid. */
  431                 if (old == -1) {
  432                         umtxq_lock(&uq->uq_key);
  433                         umtxq_busy(&uq->uq_key);
  434                         umtxq_remove(uq);
  435                         umtxq_unbusy(&uq->uq_key);
  436                         umtxq_unlock(&uq->uq_key);
  437                         umtx_key_release(&uq->uq_key);
  438                         return (EFAULT);
  439                 }
  440 
  441                 /*
  442                  * We set the contested bit, sleep. Otherwise the lock changed
  443                  * and we need to retry or we lost a race to the thread
  444                  * unlocking the umtx.
  445                  */
  446                 umtxq_lock(&uq->uq_key);
  447                 if (old == owner && (uq->uq_flags & UQF_UMTXQ)) {
  448                         error = umtxq_sleep(td, &uq->uq_key, PCATCH,
  449                                        "umtx", timo);
  450                 }
  451                 umtxq_busy(&uq->uq_key);
  452                 umtxq_remove(uq);
  453                 umtxq_unbusy(&uq->uq_key);
  454                 umtxq_unlock(&uq->uq_key);
  455                 umtx_key_release(&uq->uq_key);
  456         }
  457 
  458         return (0);
  459 }
  460 
  461 static int
  462 do_lock(struct thread *td, struct umtx *umtx, long id,
  463         struct timespec *timeout)
  464 {
  465         struct timespec ts, ts2, ts3;
  466         struct timeval tv;
  467         int error;
  468 
  469         if (timeout == NULL) {
  470                 error = _do_lock(td, umtx, id, 0);
  471         } else {
  472                 getnanouptime(&ts);
  473                 timespecadd(&ts, timeout);
  474                 TIMESPEC_TO_TIMEVAL(&tv, timeout);
  475                 for (;;) {
  476                         error = _do_lock(td, umtx, id, tvtohz(&tv));
  477                         if (error != ETIMEDOUT)
  478                                 break;
  479                         getnanouptime(&ts2);
  480                         if (timespeccmp(&ts2, &ts, >=)) {
  481                                 error = ETIMEDOUT;
  482                                 break;
  483                         }
  484                         ts3 = ts;
  485                         timespecsub(&ts3, &ts2);
  486                         TIMESPEC_TO_TIMEVAL(&tv, &ts3);
  487                 }
  488         }
  489         /*
  490          * This lets userland back off critical region if needed.
  491          */
  492         if (error == ERESTART)
  493                 error = EINTR;
  494         return (error);
  495 }
  496 
  497 static int
  498 do_unlock(struct thread *td, struct umtx *umtx, long id)
  499 {
  500         struct umtx_key key;
  501         intptr_t owner;
  502         intptr_t old;
  503         int error;
  504         int count;
  505 
  506         /*
  507          * Make sure we own this mtx.
  508          *
  509          * XXX Need a {fu,su}ptr this is not correct on arch where
  510          * sizeof(intptr_t) != sizeof(long).
  511          */
  512         if ((owner = fuword(&umtx->u_owner)) == -1)
  513                 return (EFAULT);
  514 
  515         if ((owner & ~UMTX_CONTESTED) != id)
  516                 return (EPERM);
  517 
  518         /* We should only ever be in here for contested locks */
  519         if ((owner & UMTX_CONTESTED) == 0)
  520                 return (EINVAL);
  521 
  522         if ((error = umtx_key_get(td, umtx, &key)) != 0)
  523                 return (error);
  524 
  525         umtxq_lock(&key);
  526         umtxq_busy(&key);
  527         count = umtxq_count(&key);
  528         umtxq_unlock(&key);
  529 
  530         /*
  531          * When unlocking the umtx, it must be marked as unowned if
  532          * there is zero or one thread only waiting for it.
  533          * Otherwise, it must be marked as contested.
  534          */
  535         old = casuptr((intptr_t *)&umtx->u_owner, owner,
  536                         count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
  537         umtxq_lock(&key);
  538         umtxq_signal(&key, 0);
  539         umtxq_unbusy(&key);
  540         umtxq_unlock(&key);
  541         umtx_key_release(&key);
  542         if (old == -1)
  543                 return (EFAULT);
  544         if (old != owner)
  545                 return (EINVAL);
  546         return (0);
  547 }
  548 
  549 #ifdef COMPAT_IA32
  550 static int
  551 _do_lock32(struct thread *td, uint32_t *m, uint32_t id, int timo)
  552 {
  553         struct umtx_q *uq;
  554         int32_t owner;
  555         int32_t old;
  556         int error = 0;
  557 
  558         uq = td->td_umtxq;
  559         /*
  560          * Care must be exercised when dealing with umtx structure.  It
  561          * can fault on any access.
  562          */
  563 
  564         for (;;) {
  565                 /*
  566                  * Try the uncontested case.  This should be done in userland.
  567                  */
  568                 owner = casuword32(m, UMTX_UNOWNED, id);
  569 
  570                 /* The acquire succeeded. */
  571                 if (owner == UMTX_UNOWNED)
  572                         return (0);
  573 
  574                 /* The address was invalid. */
  575                 if (owner == -1)
  576                         return (EFAULT);
  577 
  578                 /* If no one owns it but it is contested try to acquire it. */
  579                 if (owner == UMTX_CONTESTED32) {
  580                         owner = casuword32(m,
  581                             UMTX_CONTESTED32, id | UMTX_CONTESTED32);
  582 
  583                         if (owner == UMTX_CONTESTED32)
  584                                 return (0);
  585 
  586                         /* The address was invalid. */
  587                         if (owner == -1)
  588                                 return (EFAULT);
  589 
  590                         /* If this failed the lock has changed, restart. */
  591                         continue;
  592                 }
  593 
  594                 /*
  595                  * If we caught a signal, we have retried and now
  596                  * exit immediately.
  597                  */
  598                 if (error || (error = umtxq_queue_me(td, m, uq)) != 0)
  599                         return (error);
  600 
  601                 /*
  602                  * Set the contested bit so that a release in user space
  603                  * knows to use the system call for unlock.  If this fails
  604                  * either some one else has acquired the lock or it has been
  605                  * released.
  606                  */
  607                 old = casuword32(m, owner, owner | UMTX_CONTESTED32);
  608 
  609                 /* The address was invalid. */
  610                 if (old == -1) {
  611                         umtxq_lock(&uq->uq_key);
  612                         umtxq_busy(&uq->uq_key);
  613                         umtxq_remove(uq);
  614                         umtxq_unbusy(&uq->uq_key);
  615                         umtxq_unlock(&uq->uq_key);
  616                         umtx_key_release(&uq->uq_key);
  617                         return (EFAULT);
  618                 }
  619 
  620                 /*
  621                  * We set the contested bit, sleep. Otherwise the lock changed
  622                  * and we need to retry or we lost a race to the thread
  623                  * unlocking the umtx.
  624                  */
  625                 umtxq_lock(&uq->uq_key);
  626                 if (old == owner && (uq->uq_flags & UQF_UMTXQ)) {
  627                         error = umtxq_sleep(td, &uq->uq_key, PCATCH,
  628                                        "umtx", timo);
  629                 }
  630                 umtxq_busy(&uq->uq_key);
  631                 umtxq_remove(uq);
  632                 umtxq_unbusy(&uq->uq_key);
  633                 umtxq_unlock(&uq->uq_key);
  634                 umtx_key_release(&uq->uq_key);
  635         }
  636 
  637         return (0);
  638 }
  639 
  640 static int
  641 do_lock32(struct thread *td, void *m, uint32_t id,
  642         struct timespec *timeout)
  643 {
  644         struct timespec ts, ts2, ts3;
  645         struct timeval tv;
  646         int error;
  647 
  648         if (timeout == NULL) {
  649                 error = _do_lock32(td, m, id, 0);
  650         } else {
  651                 getnanouptime(&ts);
  652                 timespecadd(&ts, timeout);
  653                 TIMESPEC_TO_TIMEVAL(&tv, timeout);
  654                 for (;;) {
  655                         error = _do_lock32(td, m, id, tvtohz(&tv));
  656                         if (error != ETIMEDOUT)
  657                                 break;
  658                         getnanouptime(&ts2);
  659                         if (timespeccmp(&ts2, &ts, >=)) {
  660                                 error = ETIMEDOUT;
  661                                 break;
  662                         }
  663                         ts3 = ts;
  664                         timespecsub(&ts3, &ts2);
  665                         TIMESPEC_TO_TIMEVAL(&tv, &ts3);
  666                 }
  667         }
  668         /*
  669          * This lets userland back off critical region if needed.
  670          */
  671         if (error == ERESTART)
  672                 error = EINTR;
  673         return (error);
  674 }
  675 
  676 static int
  677 do_unlock32(struct thread *td, uint32_t *m, uint32_t id)
  678 {
  679         struct umtx_key key;
  680         int32_t owner;
  681         int32_t old;
  682         int error;
  683         int count;
  684 
  685         /*
  686          * Make sure we own this mtx.
  687          *
  688          * XXX Need a {fu,su}ptr this is not correct on arch where
  689          * sizeof(intptr_t) != sizeof(long).
  690          */
  691         if ((owner = fuword32(m)) == -1)
  692                 return (EFAULT);
  693 
  694         if ((owner & ~UMTX_CONTESTED32) != id)
  695                 return (EPERM);
  696 
  697         /* We should only ever be in here for contested locks */
  698         if ((owner & UMTX_CONTESTED32) == 0)
  699                 return (EINVAL);
  700 
  701         if ((error = umtx_key_get(td, m, &key)) != 0)
  702                 return (error);
  703 
  704         umtxq_lock(&key);
  705         umtxq_busy(&key);
  706         count = umtxq_count(&key);
  707         umtxq_unlock(&key);
  708 
  709         /*
  710          * When unlocking the umtx, it must be marked as unowned if
  711          * there is zero or one thread only waiting for it.
  712          * Otherwise, it must be marked as contested.
  713          */
  714         old = casuword32(m, owner,
  715                         count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED32);
  716         umtxq_lock(&key);
  717         umtxq_signal(&key, 0);
  718         umtxq_unbusy(&key);
  719         umtxq_unlock(&key);
  720         umtx_key_release(&key);
  721         if (old == -1)
  722                 return (EFAULT);
  723         if (old != owner)
  724                 return (EINVAL);
  725         return (0);
  726 }
  727 #endif
  728 
  729 static int
  730 do_wait(struct thread *td, struct umtx *umtx, long id, struct timespec *timeout,
  731         int compat32)
  732 {
  733         struct umtx_q *uq;
  734         struct timespec ts, ts2, ts3;
  735         struct timeval tv;
  736         long tmp;
  737         int error = 0;
  738 
  739         uq = td->td_umtxq;
  740         if ((error = umtxq_queue_me(td, umtx, uq)) != 0)
  741                 return (error);
  742         if (compat32 == 0)
  743                 tmp = fuword(&umtx->u_owner);
  744         else
  745                 tmp = fuword32(&umtx->u_owner);
  746         umtxq_lock(&uq->uq_key);
  747         if (tmp != id) {
  748                 umtxq_remove(uq);
  749         } else if (timeout == NULL) {
  750                 if (uq->uq_flags & UQF_UMTXQ)
  751                         error = umtxq_sleep(td, &uq->uq_key,
  752                             PCATCH, "ucond", 0);
  753         } else {
  754                 getnanouptime(&ts);
  755                 timespecadd(&ts, timeout);
  756                 TIMESPEC_TO_TIMEVAL(&tv, timeout);
  757                 for (;;) {
  758                         if (uq->uq_flags & UQF_UMTXQ) {
  759                                 error = umtxq_sleep(td, &uq->uq_key, PCATCH,
  760                                             "ucondt", tvtohz(&tv));
  761                         }
  762                         if (error != ETIMEDOUT)
  763                                 break;
  764                         getnanouptime(&ts2);
  765                         if (timespeccmp(&ts2, &ts, >=)) {
  766                                 error = ETIMEDOUT;
  767                                 break;
  768                         }
  769                         ts3 = ts;
  770                         timespecsub(&ts3, &ts2);
  771                         TIMESPEC_TO_TIMEVAL(&tv, &ts3);
  772                 }
  773         }
  774         if (error != 0) {
  775                 if ((uq->uq_flags & UQF_UMTXQ) == 0) {
  776                         /*
  777                          * If we concurrently got do_cv_signal()d
  778                          * and we got an error or UNIX signals or a timeout,
  779                          * then, perform another umtxq_signal to avoid
  780                          * consuming the wakeup. This may cause supurious
  781                          * wakeup for another thread which was just queued,
  782                          * but SUSV3 explicitly allows supurious wakeup to
  783                          * occur, and indeed a kernel based implementation
  784                          * can not avoid it.
  785                          */
  786                         if (!umtxq_signal(&uq->uq_key, 1))
  787                                 error = 0;
  788                 }
  789                 if (error == ERESTART)
  790                         error = EINTR;
  791         }
  792         umtxq_remove(uq);
  793         umtxq_unlock(&uq->uq_key);
  794         umtx_key_release(&uq->uq_key);
  795         return (error);
  796 }
  797 
  798 int
  799 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake)
  800 {
  801         struct umtx_key key;
  802         int ret;
  803         
  804         if ((ret = umtx_key_get(td, uaddr, &key)) != 0)
  805                 return (ret);
  806         umtxq_lock(&key);
  807         ret = umtxq_signal(&key, n_wake);
  808         umtxq_unlock(&key);
  809         umtx_key_release(&key);
  810         return (0);
  811 }
  812 
  813 int
  814 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
  815     /* struct umtx *umtx */
  816 {
  817         return _do_lock(td, uap->umtx, td->td_tid, 0);
  818 }
  819 
  820 int
  821 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
  822     /* struct umtx *umtx */
  823 {
  824         return do_unlock(td, uap->umtx, td->td_tid);
  825 }
  826 
  827 int
  828 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
  829 {
  830         struct timespec timeout;
  831         struct timespec *ts;
  832         int error;
  833 
  834         switch(uap->op) {
  835         case UMTX_OP_LOCK:
  836                 /* Allow a null timespec (wait forever). */
  837                 if (uap->uaddr2 == NULL)
  838                         ts = NULL;
  839                 else {
  840                         error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
  841                         if (error != 0)
  842                                 break;
  843                         if (timeout.tv_nsec >= 1000000000 ||
  844                             timeout.tv_nsec < 0) {
  845                                 error = EINVAL;
  846                                 break;
  847                         }
  848                         ts = &timeout;
  849                 }
  850                 error = do_lock(td, uap->umtx, uap->id, ts);
  851                 break;
  852         case UMTX_OP_UNLOCK:
  853                 error = do_unlock(td, uap->umtx, uap->id);
  854                 break;
  855         case UMTX_OP_WAIT:
  856                 /* Allow a null timespec (wait forever). */
  857                 if (uap->uaddr2 == NULL)
  858                         ts = NULL;
  859                 else {
  860                         error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
  861                         if (error != 0)
  862                                 break;
  863                         if (timeout.tv_nsec >= 1000000000 ||
  864                             timeout.tv_nsec < 0) {
  865                                 error = EINVAL;
  866                                 break;
  867                         }
  868                         ts = &timeout;
  869                 }
  870                 error = do_wait(td, uap->umtx, uap->id, ts, 0);
  871                 break;
  872         case UMTX_OP_WAKE:
  873                 error = kern_umtx_wake(td, uap->umtx, uap->id);
  874                 break;
  875         default:
  876                 error = EINVAL;
  877                 break;
  878         }
  879         return (error);
  880 }
  881 
  882 #ifdef COMPAT_IA32
  883 int
  884 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap)
  885     /* struct umtx *umtx */
  886 {
  887         return (do_lock32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
  888 }
  889 
  890 int
  891 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap)
  892     /* struct umtx *umtx */
  893 {
  894         return (do_unlock32(td, (uint32_t *)uap->umtx, td->td_tid));
  895 }
  896 
  897 struct timespec32 {
  898         u_int32_t tv_sec;
  899         u_int32_t tv_nsec;
  900 };
  901 
  902 static inline int
  903 copyin_timeout32(void *addr, struct timespec *tsp)
  904 {
  905         struct timespec32 ts32;
  906         int error;
  907 
  908         error = copyin(addr, &ts32, sizeof(struct timespec32));
  909         if (error == 0) {
  910                 tsp->tv_sec = ts32.tv_sec;
  911                 tsp->tv_nsec = ts32.tv_nsec;
  912         }
  913         return (error);
  914 }
  915 
  916 static int
  917 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
  918 {
  919         struct timespec *ts, timeout;
  920         int error;
  921 
  922         /* Allow a null timespec (wait forever). */
  923         if (uap->uaddr2 == NULL)
  924                 ts = NULL;
  925         else {
  926                 error = copyin_timeout32(uap->uaddr2, &timeout);
  927                 if (error != 0)
  928                         return (error);
  929                 if (timeout.tv_nsec >= 1000000000 ||
  930                     timeout.tv_nsec < 0) {
  931                         return (EINVAL);
  932                 }
  933                 ts = &timeout;
  934         }
  935         return (do_lock32(td, uap->umtx, uap->id, ts));
  936 }
  937 
  938 static int
  939 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap)
  940 {
  941         return (do_unlock32(td, (uint32_t *)uap->umtx, (uint32_t)uap->id));
  942 }
  943 
  944 static int
  945 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
  946 {
  947         struct timespec *ts, timeout;
  948         int error;
  949 
  950         if (uap->uaddr2 == NULL)
  951                 ts = NULL;
  952         else {
  953                 error = copyin_timeout32(uap->uaddr2, &timeout);
  954                 if (error != 0)
  955                         return (error);
  956                 if (timeout.tv_nsec >= 1000000000 ||
  957                     timeout.tv_nsec < 0)
  958                         return (EINVAL);
  959                 ts = &timeout;
  960         }
  961         return do_wait(td, uap->umtx, uap->id, ts, 1);
  962 }
  963 
  964 int
  965 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
  966 {
  967 
  968         switch ((unsigned)uap->op) {
  969         case UMTX_OP_LOCK:
  970                 return __umtx_op_lock_umtx_compat32(td,
  971                     (struct _umtx_op_args *)uap);
  972         case UMTX_OP_UNLOCK:
  973                 return __umtx_op_unlock_umtx_compat32(td,
  974                     (struct _umtx_op_args *)uap);
  975         case UMTX_OP_WAIT:
  976                 return __umtx_op_wait_compat32(td,
  977                     (struct _umtx_op_args *)uap);
  978         case UMTX_OP_WAKE:
  979                 return kern_umtx_wake(td, (uint32_t *)uap->umtx, uap->id);
  980         default:
  981                 return (EINVAL);
  982         }
  983 }
  984 #endif
Cache object: 779e955a93f81dbcee0bb774a1b65c31
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/kern_umtx.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_umtx.c
