The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_umtx.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
    3  * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice unmodified, this list of conditions, and the following
   11  *    disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/6.0/sys/kern/kern_umtx.c 143149 2005-03-05 09:15:03Z davidxu $");
   30 
   31 #include <sys/param.h>
   32 #include <sys/kernel.h>
   33 #include <sys/limits.h>
   34 #include <sys/lock.h>
   35 #include <sys/malloc.h>
   36 #include <sys/mutex.h>
   37 #include <sys/proc.h>
   38 #include <sys/sysent.h>
   39 #include <sys/systm.h>
   40 #include <sys/sysproto.h>
   41 #include <sys/eventhandler.h>
   42 #include <sys/thr.h>
   43 #include <sys/umtx.h>
   44 
   45 #include <vm/vm.h>
   46 #include <vm/vm_param.h>
   47 #include <vm/pmap.h>
   48 #include <vm/vm_map.h>
   49 #include <vm/vm_object.h>
   50 
   51 #define UMTX_PRIVATE    0
   52 #define UMTX_SHARED     1
   53 
   54 #define UMTX_STATIC_SHARED
   55 
   56 struct umtx_key {
   57         int     type;
   58         union {
   59                 struct {
   60                         vm_object_t     object;
   61                         long            offset;
   62                 } shared;
   63                 struct {
   64                         struct umtx     *umtx;
   65                         long            pid;
   66                 } private;
   67                 struct {
   68                         void            *ptr;
   69                         long            word;
   70                 } both;
   71         } info;
   72 };
   73 
   74 struct umtx_q {
   75         LIST_ENTRY(umtx_q)      uq_next;        /* Linked list for the hash. */
   76         struct umtx_key         uq_key;         /* Umtx key. */
   77         struct thread           *uq_thread;     /* The thread waits on. */
   78         LIST_ENTRY(umtx_q)      uq_rqnext;      /* Linked list for requeuing. */
   79         vm_offset_t             uq_addr;        /* Umtx's virtual address. */
   80 };
   81 
   82 LIST_HEAD(umtx_head, umtx_q);
   83 struct umtxq_chain {
   84         struct mtx              uc_lock;        /* Lock for this chain. */
   85         struct umtx_head        uc_queue;       /* List of sleep queues. */
   86 #define UCF_BUSY                0x01
   87 #define UCF_WANT                0x02
   88         int                     uc_flags;
   89 };
   90 
   91 #define GOLDEN_RATIO_PRIME      2654404609U
   92 #define UMTX_CHAINS             128
   93 #define UMTX_SHIFTS             (__WORD_BIT - 7)
   94 
   95 static struct umtxq_chain umtxq_chains[UMTX_CHAINS];
   96 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
   97 
   98 static void umtxq_init_chains(void *);
   99 static int umtxq_hash(struct umtx_key *key);
  100 static struct mtx *umtxq_mtx(int chain);
  101 static void umtxq_lock(struct umtx_key *key);
  102 static void umtxq_unlock(struct umtx_key *key);
  103 static void umtxq_busy(struct umtx_key *key);
  104 static void umtxq_unbusy(struct umtx_key *key);
  105 static void umtxq_insert(struct umtx_q *uq);
  106 static void umtxq_remove(struct umtx_q *uq);
  107 static int umtxq_sleep(struct thread *td, struct umtx_key *key,
  108         int prio, const char *wmesg, int timo);
  109 static int umtxq_count(struct umtx_key *key);
  110 static int umtxq_signal(struct umtx_key *key, int nr_wakeup);
  111 #ifdef UMTX_DYNAMIC_SHARED
  112 static void fork_handler(void *arg, struct proc *p1, struct proc *p2,
  113         int flags);
  114 #endif
  115 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2);
  116 static int umtx_key_get(struct thread *td, struct umtx *umtx,
  117         struct umtx_key *key);
  118 static void umtx_key_release(struct umtx_key *key);
  119 
  120 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_init_chains, NULL);
  121 
  122 struct umtx_q *
  123 umtxq_alloc(void)
  124 {
  125         return (malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK));
  126 }
  127 
  128 void
  129 umtxq_free(struct umtx_q *uq)
  130 {
  131         free(uq, M_UMTX);
  132 }
  133 
  134 static void
  135 umtxq_init_chains(void *arg __unused)
  136 {
  137         int i;
  138 
  139         for (i = 0; i < UMTX_CHAINS; ++i) {
  140                 mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL,
  141                          MTX_DEF | MTX_DUPOK);
  142                 LIST_INIT(&umtxq_chains[i].uc_queue);
  143                 umtxq_chains[i].uc_flags = 0;
  144         }
  145 #ifdef UMTX_DYNAMIC_SHARED
  146         EVENTHANDLER_REGISTER(process_fork, fork_handler, 0, 10000);
  147 #endif
  148 }
  149 
  150 static inline int
  151 umtxq_hash(struct umtx_key *key)
  152 {
  153         unsigned n = (uintptr_t)key->info.both.ptr + key->info.both.word;
  154         return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS);
  155 }
  156 
  157 static inline int
  158 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
  159 {
  160         return (k1->type == k2->type &&
  161                 k1->info.both.ptr == k2->info.both.ptr &&
  162                 k1->info.both.word == k2->info.both.word);
  163 }
  164 
  165 static inline struct mtx *
  166 umtxq_mtx(int chain)
  167 {
  168         return (&umtxq_chains[chain].uc_lock);
  169 }
  170 
  171 static inline void
  172 umtxq_busy(struct umtx_key *key)
  173 {
  174         int chain = umtxq_hash(key);
  175 
  176         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  177         while (umtxq_chains[chain].uc_flags & UCF_BUSY) {
  178                 umtxq_chains[chain].uc_flags |= UCF_WANT;
  179                 msleep(&umtxq_chains[chain], umtxq_mtx(chain),
  180                        curthread->td_priority, "umtxq_busy", 0);
  181         }
  182         umtxq_chains[chain].uc_flags |= UCF_BUSY;
  183 }
  184 
  185 static inline void
  186 umtxq_unbusy(struct umtx_key *key)
  187 {
  188         int chain = umtxq_hash(key);
  189 
  190         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  191         KASSERT(umtxq_chains[chain].uc_flags & UCF_BUSY, ("not busy"));
  192         umtxq_chains[chain].uc_flags &= ~UCF_BUSY;
  193         if (umtxq_chains[chain].uc_flags & UCF_WANT) {
  194                 umtxq_chains[chain].uc_flags &= ~UCF_WANT;
  195                 wakeup(&umtxq_chains[chain]);
  196         }
  197 }
  198 
  199 static inline void
  200 umtxq_lock(struct umtx_key *key)
  201 {
  202         int chain = umtxq_hash(key);
  203         mtx_lock(umtxq_mtx(chain));
  204 }
  205 
  206 static inline void
  207 umtxq_unlock(struct umtx_key *key)
  208 {
  209         int chain = umtxq_hash(key);
  210         mtx_unlock(umtxq_mtx(chain));
  211 }
  212 
  213 /*
  214  * Insert a thread onto the umtx queue.
  215  */
  216 static inline void
  217 umtxq_insert(struct umtx_q *uq)
  218 {
  219         struct umtx_head *head;
  220         int chain = umtxq_hash(&uq->uq_key);
  221 
  222         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  223         head = &umtxq_chains[chain].uc_queue;
  224         LIST_INSERT_HEAD(head, uq, uq_next);
  225         mtx_lock_spin(&sched_lock);
  226         uq->uq_thread->td_flags |= TDF_UMTXQ;
  227         mtx_unlock_spin(&sched_lock);
  228 }
  229 
  230 /*
  231  * Remove thread from the umtx queue.
  232  */
  233 static inline void
  234 umtxq_remove(struct umtx_q *uq)
  235 {
  236         mtx_assert(umtxq_mtx(umtxq_hash(&uq->uq_key)), MA_OWNED);
  237         if (uq->uq_thread->td_flags & TDF_UMTXQ) {
  238                 LIST_REMOVE(uq, uq_next);
  239                 /* turning off TDF_UMTXQ should be the last thing. */
  240                 mtx_lock_spin(&sched_lock);
  241                 uq->uq_thread->td_flags &= ~TDF_UMTXQ;
  242                 mtx_unlock_spin(&sched_lock);
  243         }
  244 }
  245 
  246 static int
  247 umtxq_count(struct umtx_key *key)
  248 {
  249         struct umtx_q *uq;
  250         struct umtx_head *head;
  251         int chain, count = 0;
  252 
  253         chain = umtxq_hash(key);
  254         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  255         head = &umtxq_chains[chain].uc_queue;
  256         LIST_FOREACH(uq, head, uq_next) {
  257                 if (umtx_key_match(&uq->uq_key, key)) {
  258                         if (++count > 1)
  259                                 break;
  260                 }
  261         }
  262         return (count);
  263 }
  264 
  265 static int
  266 umtxq_signal(struct umtx_key *key, int n_wake)
  267 {
  268         struct umtx_q *uq, *next;
  269         struct umtx_head *head;
  270         struct thread *blocked = NULL;
  271         int chain, ret;
  272 
  273         ret = 0;
  274         chain = umtxq_hash(key);
  275         mtx_assert(umtxq_mtx(chain), MA_OWNED);
  276         head = &umtxq_chains[chain].uc_queue;
  277         for (uq = LIST_FIRST(head); uq; uq = next) {
  278                 next = LIST_NEXT(uq, uq_next);
  279                 if (umtx_key_match(&uq->uq_key, key)) {
  280                         blocked = uq->uq_thread;
  281                         umtxq_remove(uq);
  282                         wakeup(blocked);
  283                         if (++ret >= n_wake)
  284                                 break;
  285                 }
  286         }
  287         return (ret);
  288 }
  289 
  290 static inline int
  291 umtxq_sleep(struct thread *td, struct umtx_key *key, int priority,
  292             const char *wmesg, int timo)
  293 {
  294         int chain = umtxq_hash(key);
  295         int error = msleep(td, umtxq_mtx(chain), priority, wmesg, timo);
  296         if (error == EWOULDBLOCK)
  297                 error = ETIMEDOUT;
  298         return (error);
  299 }
  300 
  301 static int
  302 umtx_key_get(struct thread *td, struct umtx *umtx, struct umtx_key *key)
  303 {
  304 #if defined(UMTX_DYNAMIC_SHARED) || defined(UMTX_STATIC_SHARED)
  305         vm_map_t map;
  306         vm_map_entry_t entry;
  307         vm_pindex_t pindex;
  308         vm_prot_t prot;
  309         boolean_t wired;
  310 
  311         map = &td->td_proc->p_vmspace->vm_map;
  312         if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE,
  313             &entry, &key->info.shared.object, &pindex, &prot,
  314             &wired) != KERN_SUCCESS) {
  315                 return EFAULT;
  316         }
  317 #endif
  318 
  319 #if defined(UMTX_DYNAMIC_SHARED)
  320         key->type = UMTX_SHARED;
  321         key->info.shared.offset = entry->offset + entry->start - 
  322                 (vm_offset_t)umtx;
  323         /*
  324          * Add object reference, if we don't do this, a buggy application
  325          * deallocates the object, the object will be reused by other
  326          * applications, then unlock will wake wrong thread.
  327          */
  328         vm_object_reference(key->info.shared.object);
  329         vm_map_lookup_done(map, entry);
  330 #elif defined(UMTX_STATIC_SHARED)
  331         if (VM_INHERIT_SHARE == entry->inheritance) {
  332                 key->type = UMTX_SHARED;
  333                 key->info.shared.offset = entry->offset + entry->start -
  334                         (vm_offset_t)umtx;
  335                 vm_object_reference(key->info.shared.object);
  336         } else {
  337                 key->type = UMTX_PRIVATE;
  338                 key->info.private.umtx = umtx;
  339                 key->info.private.pid  = td->td_proc->p_pid;
  340         }
  341         vm_map_lookup_done(map, entry);
  342 #else
  343         key->type = UMTX_PRIVATE;
  344         key->info.private.umtx = umtx;
  345         key->info.private.pid  = td->td_proc->p_pid;
  346 #endif
  347         return (0);
  348 }
  349 
  350 static inline void
  351 umtx_key_release(struct umtx_key *key)
  352 {
  353         if (key->type == UMTX_SHARED)
  354                 vm_object_deallocate(key->info.shared.object);
  355 }
  356 
  357 static inline int
  358 umtxq_queue_me(struct thread *td, struct umtx *umtx, struct umtx_q *uq)
  359 {
  360         int error;
  361 
  362         if ((error = umtx_key_get(td, umtx, &uq->uq_key)) != 0)
  363                 return (error);
  364 
  365         uq->uq_addr = (vm_offset_t)umtx;
  366         uq->uq_thread = td;
  367         umtxq_lock(&uq->uq_key);
  368         /* hmm, for condition variable, we don't need busy flag. */
  369         umtxq_busy(&uq->uq_key);
  370         umtxq_insert(uq);
  371         umtxq_unbusy(&uq->uq_key);
  372         umtxq_unlock(&uq->uq_key);
  373         return (0);
  374 }
  375 
  376 #if defined(UMTX_DYNAMIC_SHARED)
  377 static void
  378 fork_handler(void *arg, struct proc *p1, struct proc *p2, int flags)
  379 {
  380         vm_map_t map;
  381         vm_map_entry_t entry;
  382         vm_object_t object;
  383         vm_pindex_t pindex;
  384         vm_prot_t prot;
  385         boolean_t wired;
  386         struct umtx_key key;
  387         LIST_HEAD(, umtx_q) workq;
  388         struct umtx_q *uq;
  389         struct thread *td;
  390         int onq;
  391 
  392         LIST_INIT(&workq);
  393 
  394         /* Collect threads waiting on umtxq */
  395         PROC_LOCK(p1);
  396         FOREACH_THREAD_IN_PROC(p1, td) {
  397                 if (td->td_flags & TDF_UMTXQ) {
  398                         uq = td->td_umtxq;
  399                         if (uq)
  400                                 LIST_INSERT_HEAD(&workq, uq, uq_rqnext);
  401                 }
  402         }
  403         PROC_UNLOCK(p1);
  404 
  405         LIST_FOREACH(uq, &workq, uq_rqnext) {
  406                 map = &p1->p_vmspace->vm_map;
  407                 if (vm_map_lookup(&map, uq->uq_addr, VM_PROT_WRITE,
  408                     &entry, &object, &pindex, &prot, &wired) != KERN_SUCCESS) {
  409                         continue;
  410                 }
  411                 key.type = UMTX_SHARED;
  412                 key.info.shared.object = object;
  413                 key.info.shared.offset = entry->offset + entry->start -
  414                         uq->uq_addr;
  415                 if (umtx_key_match(&key, &uq->uq_key)) {
  416                         vm_map_lookup_done(map, entry);
  417                         continue;
  418                 }
  419                 
  420                 umtxq_lock(&uq->uq_key);
  421                 umtxq_busy(&uq->uq_key);
  422                 if (uq->uq_thread->td_flags & TDF_UMTXQ) {
  423                         umtxq_remove(uq);
  424                         onq = 1;
  425                 } else
  426                         onq = 0;
  427                 umtxq_unbusy(&uq->uq_key);
  428                 umtxq_unlock(&uq->uq_key);
  429                 if (onq) {
  430                         vm_object_deallocate(uq->uq_key.info.shared.object);
  431                         uq->uq_key = key;
  432                         umtxq_lock(&uq->uq_key);
  433                         umtxq_busy(&uq->uq_key);
  434                         umtxq_insert(uq);
  435                         umtxq_unbusy(&uq->uq_key);
  436                         umtxq_unlock(&uq->uq_key);
  437                         vm_object_reference(uq->uq_key.info.shared.object);
  438                 }
  439                 vm_map_lookup_done(map, entry);
  440         }
  441 }
  442 #endif
  443 
  444 static int
  445 _do_lock(struct thread *td, struct umtx *umtx, long id, int timo)
  446 {
  447         struct umtx_q *uq;
  448         intptr_t owner;
  449         intptr_t old;
  450         int error = 0;
  451 
  452         uq = td->td_umtxq;
  453         /*
  454          * Care must be exercised when dealing with umtx structure.  It
  455          * can fault on any access.
  456          */
  457 
  458         for (;;) {
  459                 /*
  460                  * Try the uncontested case.  This should be done in userland.
  461                  */
  462                 owner = casuptr((intptr_t *)&umtx->u_owner,
  463                     UMTX_UNOWNED, id);
  464 
  465                 /* The acquire succeeded. */
  466                 if (owner == UMTX_UNOWNED)
  467                         return (0);
  468 
  469                 /* The address was invalid. */
  470                 if (owner == -1)
  471                         return (EFAULT);
  472 
  473                 /* If no one owns it but it is contested try to acquire it. */
  474                 if (owner == UMTX_CONTESTED) {
  475                         owner = casuptr((intptr_t *)&umtx->u_owner,
  476                             UMTX_CONTESTED, id | UMTX_CONTESTED);
  477 
  478                         if (owner == UMTX_CONTESTED)
  479                                 return (0);
  480 
  481                         /* The address was invalid. */
  482                         if (owner == -1)
  483                                 return (EFAULT);
  484 
  485                         /* If this failed the lock has changed, restart. */
  486                         continue;
  487                 }
  488 
  489                 /*
  490                  * If we caught a signal, we have retried and now
  491                  * exit immediately.
  492                  */
  493                 if (error || (error = umtxq_queue_me(td, umtx, uq)) != 0)
  494                         return (error);
  495 
  496                 /*
  497                  * Set the contested bit so that a release in user space
  498                  * knows to use the system call for unlock.  If this fails
  499                  * either some one else has acquired the lock or it has been
  500                  * released.
  501                  */
  502                 old = casuptr((intptr_t *)&umtx->u_owner, owner,
  503                     owner | UMTX_CONTESTED);
  504 
  505                 /* The address was invalid. */
  506                 if (old == -1) {
  507                         umtxq_lock(&uq->uq_key);
  508                         umtxq_busy(&uq->uq_key);
  509                         umtxq_remove(uq);
  510                         umtxq_unbusy(&uq->uq_key);
  511                         umtxq_unlock(&uq->uq_key);
  512                         umtx_key_release(&uq->uq_key);
  513                         return (EFAULT);
  514                 }
  515 
  516                 /*
  517                  * We set the contested bit, sleep. Otherwise the lock changed
  518                  * and we need to retry or we lost a race to the thread
  519                  * unlocking the umtx.
  520                  */
  521                 umtxq_lock(&uq->uq_key);
  522                 if (old == owner && (td->td_flags & TDF_UMTXQ)) {
  523                         error = umtxq_sleep(td, &uq->uq_key,
  524                                        td->td_priority | PCATCH,
  525                                        "umtx", timo);
  526                 }
  527                 umtxq_busy(&uq->uq_key);
  528                 umtxq_remove(uq);
  529                 umtxq_unbusy(&uq->uq_key);
  530                 umtxq_unlock(&uq->uq_key);
  531                 umtx_key_release(&uq->uq_key);
  532         }
  533 
  534         return (0);
  535 }
  536 
  537 static int
  538 do_lock(struct thread *td, struct umtx *umtx, long id,
  539         struct timespec *timeout)
  540 {
  541         struct timespec ts, ts2, ts3;
  542         struct timeval tv;
  543         int error;
  544 
  545         if (timeout == NULL) {
  546                 error = _do_lock(td, umtx, id, 0);
  547         } else {
  548                 getnanouptime(&ts);
  549                 timespecadd(&ts, timeout);
  550                 TIMESPEC_TO_TIMEVAL(&tv, timeout);
  551                 for (;;) {
  552                         error = _do_lock(td, umtx, id, tvtohz(&tv));
  553                         if (error != ETIMEDOUT)
  554                                 break;
  555                         getnanouptime(&ts2);
  556                         if (timespeccmp(&ts2, &ts, >=)) {
  557                                 error = ETIMEDOUT;
  558                                 break;
  559                         }
  560                         ts3 = ts;
  561                         timespecsub(&ts3, &ts2);
  562                         TIMESPEC_TO_TIMEVAL(&tv, &ts3);
  563                 }
  564         }
  565         /*
  566          * This lets userland back off critical region if needed.
  567          */
  568         if (error == ERESTART)
  569                 error = EINTR;
  570         return (error);
  571 }
  572 
  573 static int
  574 do_unlock(struct thread *td, struct umtx *umtx, long id)
  575 {
  576         struct umtx_key key;
  577         intptr_t owner;
  578         intptr_t old;
  579         int error;
  580         int count;
  581 
  582         /*
  583          * Make sure we own this mtx.
  584          *
  585          * XXX Need a {fu,su}ptr this is not correct on arch where
  586          * sizeof(intptr_t) != sizeof(long).
  587          */
  588         if ((owner = fuword(&umtx->u_owner)) == -1)
  589                 return (EFAULT);
  590 
  591         if ((owner & ~UMTX_CONTESTED) != id)
  592                 return (EPERM);
  593 
  594         /* We should only ever be in here for contested locks */
  595         if ((owner & UMTX_CONTESTED) == 0)
  596                 return (EINVAL);
  597 
  598         if ((error = umtx_key_get(td, umtx, &key)) != 0)
  599                 return (error);
  600 
  601         umtxq_lock(&key);
  602         umtxq_busy(&key);
  603         count = umtxq_count(&key);
  604         umtxq_unlock(&key);
  605 
  606         /*
  607          * When unlocking the umtx, it must be marked as unowned if
  608          * there is zero or one thread only waiting for it.
  609          * Otherwise, it must be marked as contested.
  610          */
  611         old = casuptr((intptr_t *)&umtx->u_owner, owner,
  612                         count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
  613         umtxq_lock(&key);
  614         umtxq_signal(&key, 0);
  615         umtxq_unbusy(&key);
  616         umtxq_unlock(&key);
  617         umtx_key_release(&key);
  618         if (old == -1)
  619                 return (EFAULT);
  620         if (old != owner)
  621                 return (EINVAL);
  622         return (0);
  623 }
  624 
  625 static int
  626 do_wait(struct thread *td, struct umtx *umtx, long id, struct timespec *timeout)
  627 {
  628         struct umtx_q *uq;
  629         struct timespec ts, ts2, ts3;
  630         struct timeval tv;
  631         long tmp;
  632         int error = 0;
  633 
  634         uq = td->td_umtxq;
  635         if ((error = umtxq_queue_me(td, umtx, uq)) != 0)
  636                 return (error);
  637         tmp = fuword(&umtx->u_owner);
  638         if (tmp != id) {
  639                 umtxq_lock(&uq->uq_key);
  640                 umtxq_remove(uq);
  641                 umtxq_unlock(&uq->uq_key);
  642         } else if (timeout == NULL) {
  643                 umtxq_lock(&uq->uq_key);
  644                 if (td->td_flags & TDF_UMTXQ)
  645                         error = umtxq_sleep(td, &uq->uq_key,
  646                                td->td_priority | PCATCH, "ucond", 0);
  647                 if (!(td->td_flags & TDF_UMTXQ))
  648                         error = 0;
  649                 else
  650                         umtxq_remove(uq);
  651                 umtxq_unlock(&uq->uq_key);
  652         } else {
  653                 getnanouptime(&ts);
  654                 timespecadd(&ts, timeout);
  655                 TIMESPEC_TO_TIMEVAL(&tv, timeout);
  656                 for (;;) {
  657                         umtxq_lock(&uq->uq_key);
  658                         if (td->td_flags & TDF_UMTXQ) {
  659                                 error = umtxq_sleep(td, &uq->uq_key,
  660                                             td->td_priority | PCATCH,
  661                                             "ucond", tvtohz(&tv));
  662                         }
  663                         if (!(td->td_flags & TDF_UMTXQ)) {
  664                                 umtxq_unlock(&uq->uq_key);
  665                                 goto out;
  666                         }
  667                         umtxq_unlock(&uq->uq_key);
  668                         if (error != ETIMEDOUT)
  669                                 break;
  670                         getnanouptime(&ts2);
  671                         if (timespeccmp(&ts2, &ts, >=)) {
  672                                 error = ETIMEDOUT;
  673                                 break;
  674                         }
  675                         ts3 = ts;
  676                         timespecsub(&ts3, &ts2);
  677                         TIMESPEC_TO_TIMEVAL(&tv, &ts3);
  678                 }
  679                 umtxq_lock(&uq->uq_key);
  680                 umtxq_remove(uq);
  681                 umtxq_unlock(&uq->uq_key);
  682         }
  683 out:
  684         umtx_key_release(&uq->uq_key);
  685         if (error == ERESTART)
  686                 error = EINTR;
  687         return (error);
  688 }
  689 
  690 static int
  691 do_wake(struct thread *td, void *uaddr, int n_wake)
  692 {
  693         struct umtx_key key;
  694         int ret;
  695         
  696         if ((ret = umtx_key_get(td, uaddr, &key)) != 0)
  697                 return (ret);
  698         umtxq_lock(&key);
  699         ret = umtxq_signal(&key, n_wake);
  700         umtxq_unlock(&key);
  701         umtx_key_release(&key);
  702         return (0);
  703 }
  704 
  705 int
  706 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
  707     /* struct umtx *umtx */
  708 {
  709         return _do_lock(td, uap->umtx, td->td_tid, 0);
  710 }
  711 
  712 int
  713 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
  714     /* struct umtx *umtx */
  715 {
  716         return do_unlock(td, uap->umtx, td->td_tid);
  717 }
  718 
  719 int
  720 _umtx_op(struct thread *td, struct _umtx_op_args *uap)
  721 {
  722         struct timespec timeout;
  723         struct timespec *ts;
  724         int error;
  725 
  726         switch(uap->op) {
  727         case UMTX_OP_LOCK:
  728                 /* Allow a null timespec (wait forever). */
  729                 if (uap->uaddr2 == NULL)
  730                         ts = NULL;
  731                 else {
  732                         error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
  733                         if (error != 0)
  734                                 break;
  735                         if (timeout.tv_nsec >= 1000000000 ||
  736                             timeout.tv_nsec < 0) {
  737                                 error = EINVAL;
  738                                 break;
  739                         }
  740                         ts = &timeout;
  741                 }
  742                 error = do_lock(td, uap->umtx, uap->id, ts);
  743                 break;
  744         case UMTX_OP_UNLOCK:
  745                 error = do_unlock(td, uap->umtx, uap->id);
  746                 break;
  747         case UMTX_OP_WAIT:
  748                 /* Allow a null timespec (wait forever). */
  749                 if (uap->uaddr2 == NULL)
  750                         ts = NULL;
  751                 else {
  752                         error = copyin(uap->uaddr2, &timeout, sizeof(timeout));
  753                         if (error != 0)
  754                                 break;
  755                         if (timeout.tv_nsec >= 1000000000 ||
  756                             timeout.tv_nsec < 0) {
  757                                 error = EINVAL;
  758                                 break;
  759                         }
  760                         ts = &timeout;
  761                 }
  762                 error = do_wait(td, uap->umtx, uap->id, ts);
  763                 break;
  764         case UMTX_OP_WAKE:
  765                 error = do_wake(td, uap->umtx, uap->id);
  766                 break;
  767         default:
  768                 error = EINVAL;
  769                 break;
  770         }
  771         return (error);
  772 }

Cache object: 9a3644afbd817b897613c60b31312089


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.