The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_umtx.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2015, 2016 The FreeBSD Foundation
    3  * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
    4  * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by Konstantin Belousov
    8  * under sponsorship from the FreeBSD Foundation.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice unmodified, this list of conditions, and the following
   15  *    disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include "opt_compat.h"
   36 #include "opt_umtx_profiling.h"
   37 
   38 #include <sys/param.h>
   39 #include <sys/kernel.h>
   40 #include <sys/fcntl.h>
   41 #include <sys/file.h>
   42 #include <sys/filedesc.h>
   43 #include <sys/limits.h>
   44 #include <sys/lock.h>
   45 #include <sys/malloc.h>
   46 #include <sys/mman.h>
   47 #include <sys/mutex.h>
   48 #include <sys/priv.h>
   49 #include <sys/proc.h>
   50 #include <sys/resource.h>
   51 #include <sys/resourcevar.h>
   52 #include <sys/rwlock.h>
   53 #include <sys/sbuf.h>
   54 #include <sys/sched.h>
   55 #include <sys/smp.h>
   56 #include <sys/sysctl.h>
   57 #include <sys/sysent.h>
   58 #include <sys/systm.h>
   59 #include <sys/sysproto.h>
   60 #include <sys/syscallsubr.h>
   61 #include <sys/taskqueue.h>
   62 #include <sys/time.h>
   63 #include <sys/eventhandler.h>
   64 #include <sys/umtx.h>
   65 
   66 #include <security/mac/mac_framework.h>
   67 
   68 #include <vm/vm.h>
   69 #include <vm/vm_param.h>
   70 #include <vm/pmap.h>
   71 #include <vm/vm_map.h>
   72 #include <vm/vm_object.h>
   73 
   74 #include <machine/atomic.h>
   75 #include <machine/cpu.h>
   76 
   77 #ifdef COMPAT_FREEBSD32
   78 #include <compat/freebsd32/freebsd32_proto.h>
   79 #endif
   80 
   81 #define _UMUTEX_TRY             1
   82 #define _UMUTEX_WAIT            2
   83 
   84 #ifdef UMTX_PROFILING
   85 #define UPROF_PERC_BIGGER(w, f, sw, sf)                                 \
   86         (((w) > (sw)) || ((w) == (sw) && (f) > (sf)))
   87 #endif
   88 
   89 /* Priority inheritance mutex info. */
   90 struct umtx_pi {
   91         /* Owner thread */
   92         struct thread           *pi_owner;
   93 
   94         /* Reference count */
   95         int                     pi_refcount;
   96 
   97         /* List entry to link umtx holding by thread */
   98         TAILQ_ENTRY(umtx_pi)    pi_link;
   99 
  100         /* List entry in hash */
  101         TAILQ_ENTRY(umtx_pi)    pi_hashlink;
  102 
  103         /* List for waiters */
  104         TAILQ_HEAD(,umtx_q)     pi_blocked;
  105 
  106         /* Identify a userland lock object */
  107         struct umtx_key         pi_key;
  108 };
  109 
  110 /* A userland synchronous object user. */
  111 struct umtx_q {
  112         /* Linked list for the hash. */
  113         TAILQ_ENTRY(umtx_q)     uq_link;
  114 
  115         /* Umtx key. */
  116         struct umtx_key         uq_key;
  117 
  118         /* Umtx flags. */
  119         int                     uq_flags;
  120 #define UQF_UMTXQ       0x0001
  121 
  122         /* The thread waits on. */
  123         struct thread           *uq_thread;
  124 
  125         /*
  126          * Blocked on PI mutex. read can use chain lock
  127          * or umtx_lock, write must have both chain lock and
  128          * umtx_lock being hold.
  129          */
  130         struct umtx_pi          *uq_pi_blocked;
  131 
  132         /* On blocked list */
  133         TAILQ_ENTRY(umtx_q)     uq_lockq;
  134 
  135         /* Thread contending with us */
  136         TAILQ_HEAD(,umtx_pi)    uq_pi_contested;
  137 
  138         /* Inherited priority from PP mutex */
  139         u_char                  uq_inherited_pri;
  140 
  141         /* Spare queue ready to be reused */
  142         struct umtxq_queue      *uq_spare_queue;
  143 
  144         /* The queue we on */
  145         struct umtxq_queue      *uq_cur_queue;
  146 };
  147 
  148 TAILQ_HEAD(umtxq_head, umtx_q);
  149 
  150 /* Per-key wait-queue */
  151 struct umtxq_queue {
  152         struct umtxq_head       head;
  153         struct umtx_key         key;
  154         LIST_ENTRY(umtxq_queue) link;
  155         int                     length;
  156 };
  157 
  158 LIST_HEAD(umtxq_list, umtxq_queue);
  159 
  160 /* Userland lock object's wait-queue chain */
  161 struct umtxq_chain {
  162         /* Lock for this chain. */
  163         struct mtx              uc_lock;
  164 
  165         /* List of sleep queues. */
  166         struct umtxq_list       uc_queue[2];
  167 #define UMTX_SHARED_QUEUE       0
  168 #define UMTX_EXCLUSIVE_QUEUE    1
  169 
  170         LIST_HEAD(, umtxq_queue) uc_spare_queue;
  171 
  172         /* Busy flag */
  173         char                    uc_busy;
  174 
  175         /* Chain lock waiters */
  176         int                     uc_waiters;
  177 
  178         /* All PI in the list */
  179         TAILQ_HEAD(,umtx_pi)    uc_pi_list;
  180 
  181 #ifdef UMTX_PROFILING
  182         u_int                   length;
  183         u_int                   max_length;
  184 #endif
  185 };
  186 
  187 #define UMTXQ_LOCKED_ASSERT(uc)         mtx_assert(&(uc)->uc_lock, MA_OWNED)
  188 
  189 /*
  190  * Don't propagate time-sharing priority, there is a security reason,
  191  * a user can simply introduce PI-mutex, let thread A lock the mutex,
  192  * and let another thread B block on the mutex, because B is
  193  * sleeping, its priority will be boosted, this causes A's priority to
  194  * be boosted via priority propagating too and will never be lowered even
  195  * if it is using 100%CPU, this is unfair to other processes.
  196  */
  197 
  198 #define UPRI(td)        (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
  199                           (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
  200                          PRI_MAX_TIMESHARE : (td)->td_user_pri)
  201 
  202 #define GOLDEN_RATIO_PRIME      2654404609U
  203 #define UMTX_CHAINS             512
  204 #define UMTX_SHIFTS             (__WORD_BIT - 9)
  205 
  206 #define GET_SHARE(flags)        \
  207     (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
  208 
  209 #define BUSY_SPINS              200
  210 
  211 struct abs_timeout {
  212         int clockid;
  213         bool is_abs_real;       /* TIMER_ABSTIME && CLOCK_REALTIME* */
  214         struct timespec cur;
  215         struct timespec end;
  216 };
  217 
  218 #ifdef COMPAT_FREEBSD32
  219 struct umutex32 {
  220         volatile __lwpid_t      m_owner;        /* Owner of the mutex */
  221         __uint32_t              m_flags;        /* Flags of the mutex */
  222         __uint32_t              m_ceilings[2];  /* Priority protect ceiling */
  223         __uint32_t              m_rb_lnk;       /* Robust linkage */
  224         __uint32_t              m_pad;
  225         __uint32_t              m_spare[2];
  226 };
  227 
  228 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32");
  229 _Static_assert(__offsetof(struct umutex, m_spare[0]) ==
  230     __offsetof(struct umutex32, m_spare[0]), "m_spare32");
  231 #endif
  232 
  233 int umtx_shm_vnobj_persistent = 0;
  234 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN,
  235     &umtx_shm_vnobj_persistent, 0,
  236     "False forces destruction of umtx attached to file, on last close");
  237 static int umtx_max_rb = 1000;
  238 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN,
  239     &umtx_max_rb, 0,
  240     "");
  241 
  242 static uma_zone_t               umtx_pi_zone;
  243 static struct umtxq_chain       umtxq_chains[2][UMTX_CHAINS];
  244 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
  245 static int                      umtx_pi_allocated;
  246 
  247 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
  248 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
  249     &umtx_pi_allocated, 0, "Allocated umtx_pi");
  250 static int umtx_verbose_rb = 1;
  251 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN,
  252     &umtx_verbose_rb, 0,
  253     "");
  254 
  255 #ifdef UMTX_PROFILING
  256 static long max_length;
  257 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
  258 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats");
  259 #endif
  260 
  261 static void abs_timeout_update(struct abs_timeout *timo);
  262 
  263 static void umtx_shm_init(void);
  264 static void umtxq_sysinit(void *);
  265 static void umtxq_hash(struct umtx_key *key);
  266 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
  267 static void umtxq_lock(struct umtx_key *key);
  268 static void umtxq_unlock(struct umtx_key *key);
  269 static void umtxq_busy(struct umtx_key *key);
  270 static void umtxq_unbusy(struct umtx_key *key);
  271 static void umtxq_insert_queue(struct umtx_q *uq, int q);
  272 static void umtxq_remove_queue(struct umtx_q *uq, int q);
  273 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *);
  274 static int umtxq_count(struct umtx_key *key);
  275 static struct umtx_pi *umtx_pi_alloc(int);
  276 static void umtx_pi_free(struct umtx_pi *pi);
  277 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags,
  278     bool rb);
  279 static void umtx_thread_cleanup(struct thread *td);
  280 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
  281     struct image_params *imgp __unused);
  282 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
  283 
  284 #define umtxq_signal(key, nwake)        umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
  285 #define umtxq_insert(uq)        umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
  286 #define umtxq_remove(uq)        umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
  287 
  288 static struct mtx umtx_lock;
  289 
  290 #ifdef UMTX_PROFILING
  291 static void
  292 umtx_init_profiling(void)
  293 {
  294         struct sysctl_oid *chain_oid;
  295         char chain_name[10];
  296         int i;
  297 
  298         for (i = 0; i < UMTX_CHAINS; ++i) {
  299                 snprintf(chain_name, sizeof(chain_name), "%d", i);
  300                 chain_oid = SYSCTL_ADD_NODE(NULL,
  301                     SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
  302                     chain_name, CTLFLAG_RD, NULL, "umtx hash stats");
  303                 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
  304                     "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
  305                 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
  306                     "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
  307         }
  308 }
  309 
  310 static int
  311 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)
  312 {
  313         char buf[512];
  314         struct sbuf sb;
  315         struct umtxq_chain *uc;
  316         u_int fract, i, j, tot, whole;
  317         u_int sf0, sf1, sf2, sf3, sf4;
  318         u_int si0, si1, si2, si3, si4;
  319         u_int sw0, sw1, sw2, sw3, sw4;
  320 
  321         sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN);
  322         for (i = 0; i < 2; i++) {
  323                 tot = 0;
  324                 for (j = 0; j < UMTX_CHAINS; ++j) {
  325                         uc = &umtxq_chains[i][j];
  326                         mtx_lock(&uc->uc_lock);
  327                         tot += uc->max_length;
  328                         mtx_unlock(&uc->uc_lock);
  329                 }
  330                 if (tot == 0)
  331                         sbuf_printf(&sb, "%u) Empty ", i);
  332                 else {
  333                         sf0 = sf1 = sf2 = sf3 = sf4 = 0;
  334                         si0 = si1 = si2 = si3 = si4 = 0;
  335                         sw0 = sw1 = sw2 = sw3 = sw4 = 0;
  336                         for (j = 0; j < UMTX_CHAINS; j++) {
  337                                 uc = &umtxq_chains[i][j];
  338                                 mtx_lock(&uc->uc_lock);
  339                                 whole = uc->max_length * 100;
  340                                 mtx_unlock(&uc->uc_lock);
  341                                 fract = (whole % tot) * 100;
  342                                 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) {
  343                                         sf0 = fract;
  344                                         si0 = j;
  345                                         sw0 = whole;
  346                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw1,
  347                                     sf1)) {
  348                                         sf1 = fract;
  349                                         si1 = j;
  350                                         sw1 = whole;
  351                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw2,
  352                                     sf2)) {
  353                                         sf2 = fract;
  354                                         si2 = j;
  355                                         sw2 = whole;
  356                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw3,
  357                                     sf3)) {
  358                                         sf3 = fract;
  359                                         si3 = j;
  360                                         sw3 = whole;
  361                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw4,
  362                                     sf4)) {
  363                                         sf4 = fract;
  364                                         si4 = j;
  365                                         sw4 = whole;
  366                                 }
  367                         }
  368                         sbuf_printf(&sb, "queue %u:\n", i);
  369                         sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot,
  370                             sf0 / tot, si0);
  371                         sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot,
  372                             sf1 / tot, si1);
  373                         sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot,
  374                             sf2 / tot, si2);
  375                         sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot,
  376                             sf3 / tot, si3);
  377                         sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot,
  378                             sf4 / tot, si4);
  379                 }
  380         }
  381         sbuf_trim(&sb);
  382         sbuf_finish(&sb);
  383         sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
  384         sbuf_delete(&sb);
  385         return (0);
  386 }
  387 
  388 static int
  389 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)
  390 {
  391         struct umtxq_chain *uc;
  392         u_int i, j;
  393         int clear, error;
  394 
  395         clear = 0;
  396         error = sysctl_handle_int(oidp, &clear, 0, req);
  397         if (error != 0 || req->newptr == NULL)
  398                 return (error);
  399 
  400         if (clear != 0) {
  401                 for (i = 0; i < 2; ++i) {
  402                         for (j = 0; j < UMTX_CHAINS; ++j) {
  403                                 uc = &umtxq_chains[i][j];
  404                                 mtx_lock(&uc->uc_lock);
  405                                 uc->length = 0;
  406                                 uc->max_length = 0;
  407                                 mtx_unlock(&uc->uc_lock);
  408                         }
  409                 }
  410         }
  411         return (0);
  412 }
  413 
  414 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear,
  415     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
  416     sysctl_debug_umtx_chains_clear, "I", "Clear umtx chains statistics");
  417 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks,
  418     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
  419     sysctl_debug_umtx_chains_peaks, "A", "Highest peaks in chains max length");
  420 #endif
  421 
  422 static void
  423 umtxq_sysinit(void *arg __unused)
  424 {
  425         int i, j;
  426 
  427         umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
  428                 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
  429         for (i = 0; i < 2; ++i) {
  430                 for (j = 0; j < UMTX_CHAINS; ++j) {
  431                         mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
  432                                  MTX_DEF | MTX_DUPOK);
  433                         LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
  434                         LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
  435                         LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
  436                         TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
  437                         umtxq_chains[i][j].uc_busy = 0;
  438                         umtxq_chains[i][j].uc_waiters = 0;
  439 #ifdef UMTX_PROFILING
  440                         umtxq_chains[i][j].length = 0;
  441                         umtxq_chains[i][j].max_length = 0;
  442 #endif
  443                 }
  444         }
  445 #ifdef UMTX_PROFILING
  446         umtx_init_profiling();
  447 #endif
  448         mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF);
  449         EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
  450             EVENTHANDLER_PRI_ANY);
  451         umtx_shm_init();
  452 }
  453 
  454 struct umtx_q *
  455 umtxq_alloc(void)
  456 {
  457         struct umtx_q *uq;
  458 
  459         uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
  460         uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
  461             M_WAITOK | M_ZERO);
  462         TAILQ_INIT(&uq->uq_spare_queue->head);
  463         TAILQ_INIT(&uq->uq_pi_contested);
  464         uq->uq_inherited_pri = PRI_MAX;
  465         return (uq);
  466 }
  467 
  468 void
  469 umtxq_free(struct umtx_q *uq)
  470 {
  471 
  472         MPASS(uq->uq_spare_queue != NULL);
  473         free(uq->uq_spare_queue, M_UMTX);
  474         free(uq, M_UMTX);
  475 }
  476 
  477 static inline void
  478 umtxq_hash(struct umtx_key *key)
  479 {
  480         unsigned n;
  481 
  482         n = (uintptr_t)key->info.both.a + key->info.both.b;
  483         key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
  484 }
  485 
  486 static inline struct umtxq_chain *
  487 umtxq_getchain(struct umtx_key *key)
  488 {
  489 
  490         if (key->type <= TYPE_SEM)
  491                 return (&umtxq_chains[1][key->hash]);
  492         return (&umtxq_chains[0][key->hash]);
  493 }
  494 
  495 /*
  496  * Lock a chain.
  497  */
  498 static inline void
  499 umtxq_lock(struct umtx_key *key)
  500 {
  501         struct umtxq_chain *uc;
  502 
  503         uc = umtxq_getchain(key);
  504         mtx_lock(&uc->uc_lock);
  505 }
  506 
  507 /*
  508  * Unlock a chain.
  509  */
  510 static inline void
  511 umtxq_unlock(struct umtx_key *key)
  512 {
  513         struct umtxq_chain *uc;
  514 
  515         uc = umtxq_getchain(key);
  516         mtx_unlock(&uc->uc_lock);
  517 }
  518 
  519 /*
  520  * Set chain to busy state when following operation
  521  * may be blocked (kernel mutex can not be used).
  522  */
  523 static inline void
  524 umtxq_busy(struct umtx_key *key)
  525 {
  526         struct umtxq_chain *uc;
  527 
  528         uc = umtxq_getchain(key);
  529         mtx_assert(&uc->uc_lock, MA_OWNED);
  530         if (uc->uc_busy) {
  531 #ifdef SMP
  532                 if (smp_cpus > 1) {
  533                         int count = BUSY_SPINS;
  534                         if (count > 0) {
  535                                 umtxq_unlock(key);
  536                                 while (uc->uc_busy && --count > 0)
  537                                         cpu_spinwait();
  538                                 umtxq_lock(key);
  539                         }
  540                 }
  541 #endif
  542                 while (uc->uc_busy) {
  543                         uc->uc_waiters++;
  544                         msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
  545                         uc->uc_waiters--;
  546                 }
  547         }
  548         uc->uc_busy = 1;
  549 }
  550 
  551 /*
  552  * Unbusy a chain.
  553  */
  554 static inline void
  555 umtxq_unbusy(struct umtx_key *key)
  556 {
  557         struct umtxq_chain *uc;
  558 
  559         uc = umtxq_getchain(key);
  560         mtx_assert(&uc->uc_lock, MA_OWNED);
  561         KASSERT(uc->uc_busy != 0, ("not busy"));
  562         uc->uc_busy = 0;
  563         if (uc->uc_waiters)
  564                 wakeup_one(uc);
  565 }
  566 
  567 static inline void
  568 umtxq_unbusy_unlocked(struct umtx_key *key)
  569 {
  570 
  571         umtxq_lock(key);
  572         umtxq_unbusy(key);
  573         umtxq_unlock(key);
  574 }
  575 
  576 static struct umtxq_queue *
  577 umtxq_queue_lookup(struct umtx_key *key, int q)
  578 {
  579         struct umtxq_queue *uh;
  580         struct umtxq_chain *uc;
  581 
  582         uc = umtxq_getchain(key);
  583         UMTXQ_LOCKED_ASSERT(uc);
  584         LIST_FOREACH(uh, &uc->uc_queue[q], link) {
  585                 if (umtx_key_match(&uh->key, key))
  586                         return (uh);
  587         }
  588 
  589         return (NULL);
  590 }
  591 
  592 static inline void
  593 umtxq_insert_queue(struct umtx_q *uq, int q)
  594 {
  595         struct umtxq_queue *uh;
  596         struct umtxq_chain *uc;
  597 
  598         uc = umtxq_getchain(&uq->uq_key);
  599         UMTXQ_LOCKED_ASSERT(uc);
  600         KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
  601         uh = umtxq_queue_lookup(&uq->uq_key, q);
  602         if (uh != NULL) {
  603                 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
  604         } else {
  605                 uh = uq->uq_spare_queue;
  606                 uh->key = uq->uq_key;
  607                 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
  608 #ifdef UMTX_PROFILING
  609                 uc->length++;
  610                 if (uc->length > uc->max_length) {
  611                         uc->max_length = uc->length;
  612                         if (uc->max_length > max_length)
  613                                 max_length = uc->max_length;
  614                 }
  615 #endif
  616         }
  617         uq->uq_spare_queue = NULL;
  618 
  619         TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
  620         uh->length++;
  621         uq->uq_flags |= UQF_UMTXQ;
  622         uq->uq_cur_queue = uh;
  623         return;
  624 }
  625 
  626 static inline void
  627 umtxq_remove_queue(struct umtx_q *uq, int q)
  628 {
  629         struct umtxq_chain *uc;
  630         struct umtxq_queue *uh;
  631 
  632         uc = umtxq_getchain(&uq->uq_key);
  633         UMTXQ_LOCKED_ASSERT(uc);
  634         if (uq->uq_flags & UQF_UMTXQ) {
  635                 uh = uq->uq_cur_queue;
  636                 TAILQ_REMOVE(&uh->head, uq, uq_link);
  637                 uh->length--;
  638                 uq->uq_flags &= ~UQF_UMTXQ;
  639                 if (TAILQ_EMPTY(&uh->head)) {
  640                         KASSERT(uh->length == 0,
  641                             ("inconsistent umtxq_queue length"));
  642 #ifdef UMTX_PROFILING
  643                         uc->length--;
  644 #endif
  645                         LIST_REMOVE(uh, link);
  646                 } else {
  647                         uh = LIST_FIRST(&uc->uc_spare_queue);
  648                         KASSERT(uh != NULL, ("uc_spare_queue is empty"));
  649                         LIST_REMOVE(uh, link);
  650                 }
  651                 uq->uq_spare_queue = uh;
  652                 uq->uq_cur_queue = NULL;
  653         }
  654 }
  655 
  656 /*
  657  * Check if there are multiple waiters
  658  */
  659 static int
  660 umtxq_count(struct umtx_key *key)
  661 {
  662         struct umtxq_chain *uc;
  663         struct umtxq_queue *uh;
  664 
  665         uc = umtxq_getchain(key);
  666         UMTXQ_LOCKED_ASSERT(uc);
  667         uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
  668         if (uh != NULL)
  669                 return (uh->length);
  670         return (0);
  671 }
  672 
  673 /*
  674  * Check if there are multiple PI waiters and returns first
  675  * waiter.
  676  */
  677 static int
  678 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
  679 {
  680         struct umtxq_chain *uc;
  681         struct umtxq_queue *uh;
  682 
  683         *first = NULL;
  684         uc = umtxq_getchain(key);
  685         UMTXQ_LOCKED_ASSERT(uc);
  686         uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
  687         if (uh != NULL) {
  688                 *first = TAILQ_FIRST(&uh->head);
  689                 return (uh->length);
  690         }
  691         return (0);
  692 }
  693 
  694 static int
  695 umtxq_check_susp(struct thread *td)
  696 {
  697         struct proc *p;
  698         int error;
  699 
  700         /*
  701          * The check for TDF_NEEDSUSPCHK is racy, but it is enough to
  702          * eventually break the lockstep loop.
  703          */
  704         if ((td->td_flags & TDF_NEEDSUSPCHK) == 0)
  705                 return (0);
  706         error = 0;
  707         p = td->td_proc;
  708         PROC_LOCK(p);
  709         if (P_SHOULDSTOP(p) ||
  710             ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) {
  711                 if (p->p_flag & P_SINGLE_EXIT)
  712                         error = EINTR;
  713                 else
  714                         error = ERESTART;
  715         }
  716         PROC_UNLOCK(p);
  717         return (error);
  718 }
  719 
  720 /*
  721  * Wake up threads waiting on an userland object.
  722  */
  723 
  724 static int
  725 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
  726 {
  727         struct umtxq_chain *uc;
  728         struct umtxq_queue *uh;
  729         struct umtx_q *uq;
  730         int ret;
  731 
  732         ret = 0;
  733         uc = umtxq_getchain(key);
  734         UMTXQ_LOCKED_ASSERT(uc);
  735         uh = umtxq_queue_lookup(key, q);
  736         if (uh != NULL) {
  737                 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
  738                         umtxq_remove_queue(uq, q);
  739                         wakeup(uq);
  740                         if (++ret >= n_wake)
  741                                 return (ret);
  742                 }
  743         }
  744         return (ret);
  745 }
  746 
  747 
  748 /*
  749  * Wake up specified thread.
  750  */
  751 static inline void
  752 umtxq_signal_thread(struct umtx_q *uq)
  753 {
  754         struct umtxq_chain *uc;
  755 
  756         uc = umtxq_getchain(&uq->uq_key);
  757         UMTXQ_LOCKED_ASSERT(uc);
  758         umtxq_remove(uq);
  759         wakeup(uq);
  760 }
  761 
  762 static inline int
  763 tstohz(const struct timespec *tsp)
  764 {
  765         struct timeval tv;
  766 
  767         TIMESPEC_TO_TIMEVAL(&tv, tsp);
  768         return tvtohz(&tv);
  769 }
  770 
  771 static void
  772 abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute,
  773         const struct timespec *timeout)
  774 {
  775 
  776         timo->clockid = clockid;
  777         if (!absolute) {
  778                 timo->is_abs_real = false;
  779                 abs_timeout_update(timo);
  780                 timo->end = timo->cur;
  781                 timespecadd(&timo->end, timeout);
  782         } else {
  783                 timo->end = *timeout;
  784                 timo->is_abs_real = clockid == CLOCK_REALTIME ||
  785                     clockid == CLOCK_REALTIME_FAST ||
  786                     clockid == CLOCK_REALTIME_PRECISE;
  787                 /*
  788                  * If is_abs_real, umtxq_sleep will read the clock
  789                  * after setting td_rtcgen; otherwise, read it here.
  790                  */
  791                 if (!timo->is_abs_real) {
  792                         abs_timeout_update(timo);
  793                 }
  794         }
  795 }
  796 
  797 static void
  798 abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime)
  799 {
  800 
  801         abs_timeout_init(timo, umtxtime->_clockid,
  802             (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
  803 }
  804 
  805 static inline void
  806 abs_timeout_update(struct abs_timeout *timo)
  807 {
  808 
  809         kern_clock_gettime(curthread, timo->clockid, &timo->cur);
  810 }
  811 
  812 static int
  813 abs_timeout_gethz(struct abs_timeout *timo)
  814 {
  815         struct timespec tts;
  816 
  817         if (timespeccmp(&timo->end, &timo->cur, <=))
  818                 return (-1);
  819         tts = timo->end;
  820         timespecsub(&tts, &timo->cur);
  821         return (tstohz(&tts));
  822 }
  823 
  824 static uint32_t
  825 umtx_unlock_val(uint32_t flags, bool rb)
  826 {
  827 
  828         if (rb)
  829                 return (UMUTEX_RB_OWNERDEAD);
  830         else if ((flags & UMUTEX_NONCONSISTENT) != 0)
  831                 return (UMUTEX_RB_NOTRECOV);
  832         else
  833                 return (UMUTEX_UNOWNED);
  834 
  835 }
  836 
  837 /*
  838  * Put thread into sleep state, before sleeping, check if
  839  * thread was removed from umtx queue.
  840  */
  841 static inline int
  842 umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime)
  843 {
  844         struct umtxq_chain *uc;
  845         int error, timo;
  846 
  847         if (abstime != NULL && abstime->is_abs_real) {
  848                 curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation);
  849                 abs_timeout_update(abstime);
  850         }
  851 
  852         uc = umtxq_getchain(&uq->uq_key);
  853         UMTXQ_LOCKED_ASSERT(uc);
  854         for (;;) {
  855                 if (!(uq->uq_flags & UQF_UMTXQ)) {
  856                         error = 0;
  857                         break;
  858                 }
  859                 if (abstime != NULL) {
  860                         timo = abs_timeout_gethz(abstime);
  861                         if (timo < 0) {
  862                                 error = ETIMEDOUT;
  863                                 break;
  864                         }
  865                 } else
  866                         timo = 0;
  867                 error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo);
  868                 if (error == EINTR || error == ERESTART) {
  869                         umtxq_lock(&uq->uq_key);
  870                         break;
  871                 }
  872                 if (abstime != NULL) {
  873                         if (abstime->is_abs_real)
  874                                 curthread->td_rtcgen =
  875                                     atomic_load_acq_int(&rtc_generation);
  876                         abs_timeout_update(abstime);
  877                 }
  878                 umtxq_lock(&uq->uq_key);
  879         }
  880 
  881         curthread->td_rtcgen = 0;
  882         return (error);
  883 }
  884 
  885 /*
  886  * Convert userspace address into unique logical address.
  887  */
  888 int
  889 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key)
  890 {
  891         struct thread *td = curthread;
  892         vm_map_t map;
  893         vm_map_entry_t entry;
  894         vm_pindex_t pindex;
  895         vm_prot_t prot;
  896         boolean_t wired;
  897 
  898         key->type = type;
  899         if (share == THREAD_SHARE) {
  900                 key->shared = 0;
  901                 key->info.private.vs = td->td_proc->p_vmspace;
  902                 key->info.private.addr = (uintptr_t)addr;
  903         } else {
  904                 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
  905                 map = &td->td_proc->p_vmspace->vm_map;
  906                 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
  907                     &entry, &key->info.shared.object, &pindex, &prot,
  908                     &wired) != KERN_SUCCESS) {
  909                         return (EFAULT);
  910                 }
  911 
  912                 if ((share == PROCESS_SHARE) ||
  913                     (share == AUTO_SHARE &&
  914                      VM_INHERIT_SHARE == entry->inheritance)) {
  915                         key->shared = 1;
  916                         key->info.shared.offset = (vm_offset_t)addr -
  917                             entry->start + entry->offset;
  918                         vm_object_reference(key->info.shared.object);
  919                 } else {
  920                         key->shared = 0;
  921                         key->info.private.vs = td->td_proc->p_vmspace;
  922                         key->info.private.addr = (uintptr_t)addr;
  923                 }
  924                 vm_map_lookup_done(map, entry);
  925         }
  926 
  927         umtxq_hash(key);
  928         return (0);
  929 }
  930 
  931 /*
  932  * Release key.
  933  */
  934 void
  935 umtx_key_release(struct umtx_key *key)
  936 {
  937         if (key->shared)
  938                 vm_object_deallocate(key->info.shared.object);
  939 }
  940 
  941 /*
  942  * Fetch and compare value, sleep on the address if value is not changed.
  943  */
  944 static int
  945 do_wait(struct thread *td, void *addr, u_long id,
  946     struct _umtx_time *timeout, int compat32, int is_private)
  947 {
  948         struct abs_timeout timo;
  949         struct umtx_q *uq;
  950         u_long tmp;
  951         uint32_t tmp32;
  952         int error = 0;
  953 
  954         uq = td->td_umtxq;
  955         if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
  956                 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
  957                 return (error);
  958 
  959         if (timeout != NULL)
  960                 abs_timeout_init2(&timo, timeout);
  961 
  962         umtxq_lock(&uq->uq_key);
  963         umtxq_insert(uq);
  964         umtxq_unlock(&uq->uq_key);
  965         if (compat32 == 0) {
  966                 error = fueword(addr, &tmp);
  967                 if (error != 0)
  968                         error = EFAULT;
  969         } else {
  970                 error = fueword32(addr, &tmp32);
  971                 if (error == 0)
  972                         tmp = tmp32;
  973                 else
  974                         error = EFAULT;
  975         }
  976         umtxq_lock(&uq->uq_key);
  977         if (error == 0) {
  978                 if (tmp == id)
  979                         error = umtxq_sleep(uq, "uwait", timeout == NULL ?
  980                             NULL : &timo);
  981                 if ((uq->uq_flags & UQF_UMTXQ) == 0)
  982                         error = 0;
  983                 else
  984                         umtxq_remove(uq);
  985         } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
  986                 umtxq_remove(uq);
  987         }
  988         umtxq_unlock(&uq->uq_key);
  989         umtx_key_release(&uq->uq_key);
  990         if (error == ERESTART)
  991                 error = EINTR;
  992         return (error);
  993 }
  994 
  995 /*
  996  * Wake up threads sleeping on the specified address.
  997  */
  998 int
  999 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
 1000 {
 1001         struct umtx_key key;
 1002         int ret;
 1003 
 1004         if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
 1005             is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
 1006                 return (ret);
 1007         umtxq_lock(&key);
 1008         umtxq_signal(&key, n_wake);
 1009         umtxq_unlock(&key);
 1010         umtx_key_release(&key);
 1011         return (0);
 1012 }
 1013 
 1014 /*
 1015  * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
 1016  */
 1017 static int
 1018 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 1019     struct _umtx_time *timeout, int mode)
 1020 {
 1021         struct abs_timeout timo;
 1022         struct umtx_q *uq;
 1023         uint32_t owner, old, id;
 1024         int error, rv;
 1025 
 1026         id = td->td_tid;
 1027         uq = td->td_umtxq;
 1028         error = 0;
 1029         if (timeout != NULL)
 1030                 abs_timeout_init2(&timo, timeout);
 1031 
 1032         /*
 1033          * Care must be exercised when dealing with umtx structure. It
 1034          * can fault on any access.
 1035          */
 1036         for (;;) {
 1037                 rv = fueword32(&m->m_owner, &owner);
 1038                 if (rv == -1)
 1039                         return (EFAULT);
 1040                 if (mode == _UMUTEX_WAIT) {
 1041                         if (owner == UMUTEX_UNOWNED ||
 1042                             owner == UMUTEX_CONTESTED ||
 1043                             owner == UMUTEX_RB_OWNERDEAD ||
 1044                             owner == UMUTEX_RB_NOTRECOV)
 1045                                 return (0);
 1046                 } else {
 1047                         /*
 1048                          * Robust mutex terminated.  Kernel duty is to
 1049                          * return EOWNERDEAD to the userspace.  The
 1050                          * umutex.m_flags UMUTEX_NONCONSISTENT is set
 1051                          * by the common userspace code.
 1052                          */
 1053                         if (owner == UMUTEX_RB_OWNERDEAD) {
 1054                                 rv = casueword32(&m->m_owner,
 1055                                     UMUTEX_RB_OWNERDEAD, &owner,
 1056                                     id | UMUTEX_CONTESTED);
 1057                                 if (rv == -1)
 1058                                         return (EFAULT);
 1059                                 if (owner == UMUTEX_RB_OWNERDEAD)
 1060                                         return (EOWNERDEAD); /* success */
 1061                                 rv = umtxq_check_susp(td);
 1062                                 if (rv != 0)
 1063                                         return (rv);
 1064                                 continue;
 1065                         }
 1066                         if (owner == UMUTEX_RB_NOTRECOV)
 1067                                 return (ENOTRECOVERABLE);
 1068 
 1069                         /*
 1070                          * Try the uncontested case.  This should be
 1071                          * done in userland.
 1072                          */
 1073                         rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
 1074                             &owner, id);
 1075                         /* The address was invalid. */
 1076                         if (rv == -1)
 1077                                 return (EFAULT);
 1078 
 1079                         /* The acquire succeeded. */
 1080                         if (owner == UMUTEX_UNOWNED)
 1081                                 return (0);
 1082 
 1083                         /*
 1084                          * If no one owns it but it is contested try
 1085                          * to acquire it.
 1086                          */
 1087                         if (owner == UMUTEX_CONTESTED) {
 1088                                 rv = casueword32(&m->m_owner,
 1089                                     UMUTEX_CONTESTED, &owner,
 1090                                     id | UMUTEX_CONTESTED);
 1091                                 /* The address was invalid. */
 1092                                 if (rv == -1)
 1093                                         return (EFAULT);
 1094 
 1095                                 if (owner == UMUTEX_CONTESTED)
 1096                                         return (0);
 1097 
 1098                                 rv = umtxq_check_susp(td);
 1099                                 if (rv != 0)
 1100                                         return (rv);
 1101 
 1102                                 /*
 1103                                  * If this failed the lock has
 1104                                  * changed, restart.
 1105                                  */
 1106                                 continue;
 1107                         }
 1108                 }
 1109 
 1110                 if (mode == _UMUTEX_TRY)
 1111                         return (EBUSY);
 1112 
 1113                 /*
 1114                  * If we caught a signal, we have retried and now
 1115                  * exit immediately.
 1116                  */
 1117                 if (error != 0)
 1118                         return (error);
 1119 
 1120                 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
 1121                     GET_SHARE(flags), &uq->uq_key)) != 0)
 1122                         return (error);
 1123 
 1124                 umtxq_lock(&uq->uq_key);
 1125                 umtxq_busy(&uq->uq_key);
 1126                 umtxq_insert(uq);
 1127                 umtxq_unlock(&uq->uq_key);
 1128 
 1129                 /*
 1130                  * Set the contested bit so that a release in user space
 1131                  * knows to use the system call for unlock.  If this fails
 1132                  * either some one else has acquired the lock or it has been
 1133                  * released.
 1134                  */
 1135                 rv = casueword32(&m->m_owner, owner, &old,
 1136                     owner | UMUTEX_CONTESTED);
 1137 
 1138                 /* The address was invalid. */
 1139                 if (rv == -1) {
 1140                         umtxq_lock(&uq->uq_key);
 1141                         umtxq_remove(uq);
 1142                         umtxq_unbusy(&uq->uq_key);
 1143                         umtxq_unlock(&uq->uq_key);
 1144                         umtx_key_release(&uq->uq_key);
 1145                         return (EFAULT);
 1146                 }
 1147 
 1148                 /*
 1149                  * We set the contested bit, sleep. Otherwise the lock changed
 1150                  * and we need to retry or we lost a race to the thread
 1151                  * unlocking the umtx.
 1152                  */
 1153                 umtxq_lock(&uq->uq_key);
 1154                 umtxq_unbusy(&uq->uq_key);
 1155                 if (old == owner)
 1156                         error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
 1157                             NULL : &timo);
 1158                 umtxq_remove(uq);
 1159                 umtxq_unlock(&uq->uq_key);
 1160                 umtx_key_release(&uq->uq_key);
 1161 
 1162                 if (error == 0)
 1163                         error = umtxq_check_susp(td);
 1164         }
 1165 
 1166         return (0);
 1167 }
 1168 
 1169 /*
 1170  * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
 1171  */
 1172 static int
 1173 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
 1174 {
 1175         struct umtx_key key;
 1176         uint32_t owner, old, id, newlock;
 1177         int error, count;
 1178 
 1179         id = td->td_tid;
 1180         /*
 1181          * Make sure we own this mtx.
 1182          */
 1183         error = fueword32(&m->m_owner, &owner);
 1184         if (error == -1)
 1185                 return (EFAULT);
 1186 
 1187         if ((owner & ~UMUTEX_CONTESTED) != id)
 1188                 return (EPERM);
 1189 
 1190         newlock = umtx_unlock_val(flags, rb);
 1191         if ((owner & UMUTEX_CONTESTED) == 0) {
 1192                 error = casueword32(&m->m_owner, owner, &old, newlock);
 1193                 if (error == -1)
 1194                         return (EFAULT);
 1195                 if (old == owner)
 1196                         return (0);
 1197                 owner = old;
 1198         }
 1199 
 1200         /* We should only ever be in here for contested locks */
 1201         if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
 1202             &key)) != 0)
 1203                 return (error);
 1204 
 1205         umtxq_lock(&key);
 1206         umtxq_busy(&key);
 1207         count = umtxq_count(&key);
 1208         umtxq_unlock(&key);
 1209 
 1210         /*
 1211          * When unlocking the umtx, it must be marked as unowned if
 1212          * there is zero or one thread only waiting for it.
 1213          * Otherwise, it must be marked as contested.
 1214          */
 1215         if (count > 1)
 1216                 newlock |= UMUTEX_CONTESTED;
 1217         error = casueword32(&m->m_owner, owner, &old, newlock);
 1218         umtxq_lock(&key);
 1219         umtxq_signal(&key, 1);
 1220         umtxq_unbusy(&key);
 1221         umtxq_unlock(&key);
 1222         umtx_key_release(&key);
 1223         if (error == -1)
 1224                 return (EFAULT);
 1225         if (old != owner)
 1226                 return (EINVAL);
 1227         return (0);
 1228 }
 1229 
 1230 /*
 1231  * Check if the mutex is available and wake up a waiter,
 1232  * only for simple mutex.
 1233  */
 1234 static int
 1235 do_wake_umutex(struct thread *td, struct umutex *m)
 1236 {
 1237         struct umtx_key key;
 1238         uint32_t owner;
 1239         uint32_t flags;
 1240         int error;
 1241         int count;
 1242 
 1243         error = fueword32(&m->m_owner, &owner);
 1244         if (error == -1)
 1245                 return (EFAULT);
 1246 
 1247         if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD &&
 1248             owner != UMUTEX_RB_NOTRECOV)
 1249                 return (0);
 1250 
 1251         error = fueword32(&m->m_flags, &flags);
 1252         if (error == -1)
 1253                 return (EFAULT);
 1254 
 1255         /* We should only ever be in here for contested locks */
 1256         if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
 1257             &key)) != 0)
 1258                 return (error);
 1259 
 1260         umtxq_lock(&key);
 1261         umtxq_busy(&key);
 1262         count = umtxq_count(&key);
 1263         umtxq_unlock(&key);
 1264 
 1265         if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD &&
 1266             owner != UMUTEX_RB_NOTRECOV) {
 1267                 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
 1268                     UMUTEX_UNOWNED);
 1269                 if (error == -1)
 1270                         error = EFAULT;
 1271         }
 1272 
 1273         umtxq_lock(&key);
 1274         if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
 1275             owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
 1276                 umtxq_signal(&key, 1);
 1277         umtxq_unbusy(&key);
 1278         umtxq_unlock(&key);
 1279         umtx_key_release(&key);
 1280         return (error);
 1281 }
 1282 
 1283 /*
 1284  * Check if the mutex has waiters and tries to fix contention bit.
 1285  */
 1286 static int
 1287 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
 1288 {
 1289         struct umtx_key key;
 1290         uint32_t owner, old;
 1291         int type;
 1292         int error;
 1293         int count;
 1294 
 1295         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT |
 1296             UMUTEX_ROBUST)) {
 1297         case 0:
 1298         case UMUTEX_ROBUST:
 1299                 type = TYPE_NORMAL_UMUTEX;
 1300                 break;
 1301         case UMUTEX_PRIO_INHERIT:
 1302                 type = TYPE_PI_UMUTEX;
 1303                 break;
 1304         case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST):
 1305                 type = TYPE_PI_ROBUST_UMUTEX;
 1306                 break;
 1307         case UMUTEX_PRIO_PROTECT:
 1308                 type = TYPE_PP_UMUTEX;
 1309                 break;
 1310         case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST):
 1311                 type = TYPE_PP_ROBUST_UMUTEX;
 1312                 break;
 1313         default:
 1314                 return (EINVAL);
 1315         }
 1316         if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0)
 1317                 return (error);
 1318 
 1319         owner = 0;
 1320         umtxq_lock(&key);
 1321         umtxq_busy(&key);
 1322         count = umtxq_count(&key);
 1323         umtxq_unlock(&key);
 1324         /*
 1325          * Only repair contention bit if there is a waiter, this means the mutex
 1326          * is still being referenced by userland code, otherwise don't update
 1327          * any memory.
 1328          */
 1329         if (count > 1) {
 1330                 error = fueword32(&m->m_owner, &owner);
 1331                 if (error == -1)
 1332                         error = EFAULT;
 1333                 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) {
 1334                         error = casueword32(&m->m_owner, owner, &old,
 1335                             owner | UMUTEX_CONTESTED);
 1336                         if (error == -1) {
 1337                                 error = EFAULT;
 1338                                 break;
 1339                         }
 1340                         if (old == owner)
 1341                                 break;
 1342                         owner = old;
 1343                         error = umtxq_check_susp(td);
 1344                         if (error != 0)
 1345                                 break;
 1346                 }
 1347         } else if (count == 1) {
 1348                 error = fueword32(&m->m_owner, &owner);
 1349                 if (error == -1)
 1350                         error = EFAULT;
 1351                 while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 &&
 1352                     (owner & UMUTEX_CONTESTED) == 0) {
 1353                         error = casueword32(&m->m_owner, owner, &old,
 1354                             owner | UMUTEX_CONTESTED);
 1355                         if (error == -1) {
 1356                                 error = EFAULT;
 1357                                 break;
 1358                         }
 1359                         if (old == owner)
 1360                                 break;
 1361                         owner = old;
 1362                         error = umtxq_check_susp(td);
 1363                         if (error != 0)
 1364                                 break;
 1365                 }
 1366         }
 1367         umtxq_lock(&key);
 1368         if (error == EFAULT) {
 1369                 umtxq_signal(&key, INT_MAX);
 1370         } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
 1371             owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
 1372                 umtxq_signal(&key, 1);
 1373         umtxq_unbusy(&key);
 1374         umtxq_unlock(&key);
 1375         umtx_key_release(&key);
 1376         return (error);
 1377 }
 1378 
 1379 static inline struct umtx_pi *
 1380 umtx_pi_alloc(int flags)
 1381 {
 1382         struct umtx_pi *pi;
 1383 
 1384         pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
 1385         TAILQ_INIT(&pi->pi_blocked);
 1386         atomic_add_int(&umtx_pi_allocated, 1);
 1387         return (pi);
 1388 }
 1389 
 1390 static inline void
 1391 umtx_pi_free(struct umtx_pi *pi)
 1392 {
 1393         uma_zfree(umtx_pi_zone, pi);
 1394         atomic_add_int(&umtx_pi_allocated, -1);
 1395 }
 1396 
 1397 /*
 1398  * Adjust the thread's position on a pi_state after its priority has been
 1399  * changed.
 1400  */
 1401 static int
 1402 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
 1403 {
 1404         struct umtx_q *uq, *uq1, *uq2;
 1405         struct thread *td1;
 1406 
 1407         mtx_assert(&umtx_lock, MA_OWNED);
 1408         if (pi == NULL)
 1409                 return (0);
 1410 
 1411         uq = td->td_umtxq;
 1412 
 1413         /*
 1414          * Check if the thread needs to be moved on the blocked chain.
 1415          * It needs to be moved if either its priority is lower than
 1416          * the previous thread or higher than the next thread.
 1417          */
 1418         uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
 1419         uq2 = TAILQ_NEXT(uq, uq_lockq);
 1420         if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
 1421             (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
 1422                 /*
 1423                  * Remove thread from blocked chain and determine where
 1424                  * it should be moved to.
 1425                  */
 1426                 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
 1427                 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
 1428                         td1 = uq1->uq_thread;
 1429                         MPASS(td1->td_proc->p_magic == P_MAGIC);
 1430                         if (UPRI(td1) > UPRI(td))
 1431                                 break;
 1432                 }
 1433 
 1434                 if (uq1 == NULL)
 1435                         TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
 1436                 else
 1437                         TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
 1438         }
 1439         return (1);
 1440 }
 1441 
 1442 static struct umtx_pi *
 1443 umtx_pi_next(struct umtx_pi *pi)
 1444 {
 1445         struct umtx_q *uq_owner;
 1446 
 1447         if (pi->pi_owner == NULL)
 1448                 return (NULL);
 1449         uq_owner = pi->pi_owner->td_umtxq;
 1450         if (uq_owner == NULL)
 1451                 return (NULL);
 1452         return (uq_owner->uq_pi_blocked);
 1453 }
 1454 
 1455 /*
 1456  * Floyd's Cycle-Finding Algorithm.
 1457  */
 1458 static bool
 1459 umtx_pi_check_loop(struct umtx_pi *pi)
 1460 {
 1461         struct umtx_pi *pi1;    /* fast iterator */
 1462 
 1463         mtx_assert(&umtx_lock, MA_OWNED);
 1464         if (pi == NULL)
 1465                 return (false);
 1466         pi1 = pi;
 1467         for (;;) {
 1468                 pi = umtx_pi_next(pi);
 1469                 if (pi == NULL)
 1470                         break;
 1471                 pi1 = umtx_pi_next(pi1);
 1472                 if (pi1 == NULL)
 1473                         break;
 1474                 pi1 = umtx_pi_next(pi1);
 1475                 if (pi1 == NULL)
 1476                         break;
 1477                 if (pi == pi1)
 1478                         return (true);
 1479         }
 1480         return (false);
 1481 }
 1482 
 1483 /*
 1484  * Propagate priority when a thread is blocked on POSIX
 1485  * PI mutex.
 1486  */
 1487 static void
 1488 umtx_propagate_priority(struct thread *td)
 1489 {
 1490         struct umtx_q *uq;
 1491         struct umtx_pi *pi;
 1492         int pri;
 1493 
 1494         mtx_assert(&umtx_lock, MA_OWNED);
 1495         pri = UPRI(td);
 1496         uq = td->td_umtxq;
 1497         pi = uq->uq_pi_blocked;
 1498         if (pi == NULL)
 1499                 return;
 1500         if (umtx_pi_check_loop(pi))
 1501                 return;
 1502 
 1503         for (;;) {
 1504                 td = pi->pi_owner;
 1505                 if (td == NULL || td == curthread)
 1506                         return;
 1507 
 1508                 MPASS(td->td_proc != NULL);
 1509                 MPASS(td->td_proc->p_magic == P_MAGIC);
 1510 
 1511                 thread_lock(td);
 1512                 if (td->td_lend_user_pri > pri)
 1513                         sched_lend_user_prio(td, pri);
 1514                 else {
 1515                         thread_unlock(td);
 1516                         break;
 1517                 }
 1518                 thread_unlock(td);
 1519 
 1520                 /*
 1521                  * Pick up the lock that td is blocked on.
 1522                  */
 1523                 uq = td->td_umtxq;
 1524                 pi = uq->uq_pi_blocked;
 1525                 if (pi == NULL)
 1526                         break;
 1527                 /* Resort td on the list if needed. */
 1528                 umtx_pi_adjust_thread(pi, td);
 1529         }
 1530 }
 1531 
 1532 /*
 1533  * Unpropagate priority for a PI mutex when a thread blocked on
 1534  * it is interrupted by signal or resumed by others.
 1535  */
 1536 static void
 1537 umtx_repropagate_priority(struct umtx_pi *pi)
 1538 {
 1539         struct umtx_q *uq, *uq_owner;
 1540         struct umtx_pi *pi2;
 1541         int pri;
 1542 
 1543         mtx_assert(&umtx_lock, MA_OWNED);
 1544 
 1545         if (umtx_pi_check_loop(pi))
 1546                 return;
 1547         while (pi != NULL && pi->pi_owner != NULL) {
 1548                 pri = PRI_MAX;
 1549                 uq_owner = pi->pi_owner->td_umtxq;
 1550 
 1551                 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
 1552                         uq = TAILQ_FIRST(&pi2->pi_blocked);
 1553                         if (uq != NULL) {
 1554                                 if (pri > UPRI(uq->uq_thread))
 1555                                         pri = UPRI(uq->uq_thread);
 1556                         }
 1557                 }
 1558 
 1559                 if (pri > uq_owner->uq_inherited_pri)
 1560                         pri = uq_owner->uq_inherited_pri;
 1561                 thread_lock(pi->pi_owner);
 1562                 sched_lend_user_prio(pi->pi_owner, pri);
 1563                 thread_unlock(pi->pi_owner);
 1564                 if ((pi = uq_owner->uq_pi_blocked) != NULL)
 1565                         umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
 1566         }
 1567 }
 1568 
 1569 /*
 1570  * Insert a PI mutex into owned list.
 1571  */
 1572 static void
 1573 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
 1574 {
 1575         struct umtx_q *uq_owner;
 1576 
 1577         uq_owner = owner->td_umtxq;
 1578         mtx_assert(&umtx_lock, MA_OWNED);
 1579         MPASS(pi->pi_owner == NULL);
 1580         pi->pi_owner = owner;
 1581         TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
 1582 }
 1583 
 1584 
 1585 /*
 1586  * Disown a PI mutex, and remove it from the owned list.
 1587  */
 1588 static void
 1589 umtx_pi_disown(struct umtx_pi *pi)
 1590 {
 1591 
 1592         mtx_assert(&umtx_lock, MA_OWNED);
 1593         TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
 1594         pi->pi_owner = NULL;
 1595 }
 1596 
 1597 /*
 1598  * Claim ownership of a PI mutex.
 1599  */
 1600 static int
 1601 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
 1602 {
 1603         struct umtx_q *uq;
 1604         int pri;
 1605 
 1606         mtx_lock(&umtx_lock);
 1607         if (pi->pi_owner == owner) {
 1608                 mtx_unlock(&umtx_lock);
 1609                 return (0);
 1610         }
 1611 
 1612         if (pi->pi_owner != NULL) {
 1613                 /*
 1614                  * userland may have already messed the mutex, sigh.
 1615                  */
 1616                 mtx_unlock(&umtx_lock);
 1617                 return (EPERM);
 1618         }
 1619         umtx_pi_setowner(pi, owner);
 1620         uq = TAILQ_FIRST(&pi->pi_blocked);
 1621         if (uq != NULL) {
 1622                 pri = UPRI(uq->uq_thread);
 1623                 thread_lock(owner);
 1624                 if (pri < UPRI(owner))
 1625                         sched_lend_user_prio(owner, pri);
 1626                 thread_unlock(owner);
 1627         }
 1628         mtx_unlock(&umtx_lock);
 1629         return (0);
 1630 }
 1631 
 1632 /*
 1633  * Adjust a thread's order position in its blocked PI mutex,
 1634  * this may result new priority propagating process.
 1635  */
 1636 void
 1637 umtx_pi_adjust(struct thread *td, u_char oldpri)
 1638 {
 1639         struct umtx_q *uq;
 1640         struct umtx_pi *pi;
 1641 
 1642         uq = td->td_umtxq;
 1643         mtx_lock(&umtx_lock);
 1644         /*
 1645          * Pick up the lock that td is blocked on.
 1646          */
 1647         pi = uq->uq_pi_blocked;
 1648         if (pi != NULL) {
 1649                 umtx_pi_adjust_thread(pi, td);
 1650                 umtx_repropagate_priority(pi);
 1651         }
 1652         mtx_unlock(&umtx_lock);
 1653 }
 1654 
 1655 /*
 1656  * Sleep on a PI mutex.
 1657  */
 1658 static int
 1659 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
 1660     const char *wmesg, struct abs_timeout *timo, bool shared)
 1661 {
 1662         struct umtxq_chain *uc;
 1663         struct thread *td, *td1;
 1664         struct umtx_q *uq1;
 1665         int error, pri;
 1666 
 1667         error = 0;
 1668         td = uq->uq_thread;
 1669         KASSERT(td == curthread, ("inconsistent uq_thread"));
 1670         uc = umtxq_getchain(&uq->uq_key);
 1671         UMTXQ_LOCKED_ASSERT(uc);
 1672         KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
 1673         umtxq_insert(uq);
 1674         mtx_lock(&umtx_lock);
 1675         if (pi->pi_owner == NULL) {
 1676                 mtx_unlock(&umtx_lock);
 1677                 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
 1678                 mtx_lock(&umtx_lock);
 1679                 if (td1 != NULL) {
 1680                         if (pi->pi_owner == NULL)
 1681                                 umtx_pi_setowner(pi, td1);
 1682                         PROC_UNLOCK(td1->td_proc);
 1683                 }
 1684         }
 1685 
 1686         TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
 1687                 pri = UPRI(uq1->uq_thread);
 1688                 if (pri > UPRI(td))
 1689                         break;
 1690         }
 1691 
 1692         if (uq1 != NULL)
 1693                 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
 1694         else
 1695                 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
 1696 
 1697         uq->uq_pi_blocked = pi;
 1698         thread_lock(td);
 1699         td->td_flags |= TDF_UPIBLOCKED;
 1700         thread_unlock(td);
 1701         umtx_propagate_priority(td);
 1702         mtx_unlock(&umtx_lock);
 1703         umtxq_unbusy(&uq->uq_key);
 1704 
 1705         error = umtxq_sleep(uq, wmesg, timo);
 1706         umtxq_remove(uq);
 1707 
 1708         mtx_lock(&umtx_lock);
 1709         uq->uq_pi_blocked = NULL;
 1710         thread_lock(td);
 1711         td->td_flags &= ~TDF_UPIBLOCKED;
 1712         thread_unlock(td);
 1713         TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
 1714         umtx_repropagate_priority(pi);
 1715         mtx_unlock(&umtx_lock);
 1716         umtxq_unlock(&uq->uq_key);
 1717 
 1718         return (error);
 1719 }
 1720 
 1721 /*
 1722  * Add reference count for a PI mutex.
 1723  */
 1724 static void
 1725 umtx_pi_ref(struct umtx_pi *pi)
 1726 {
 1727         struct umtxq_chain *uc;
 1728 
 1729         uc = umtxq_getchain(&pi->pi_key);
 1730         UMTXQ_LOCKED_ASSERT(uc);
 1731         pi->pi_refcount++;
 1732 }
 1733 
 1734 /*
 1735  * Decrease reference count for a PI mutex, if the counter
 1736  * is decreased to zero, its memory space is freed.
 1737  */
 1738 static void
 1739 umtx_pi_unref(struct umtx_pi *pi)
 1740 {
 1741         struct umtxq_chain *uc;
 1742 
 1743         uc = umtxq_getchain(&pi->pi_key);
 1744         UMTXQ_LOCKED_ASSERT(uc);
 1745         KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
 1746         if (--pi->pi_refcount == 0) {
 1747                 mtx_lock(&umtx_lock);
 1748                 if (pi->pi_owner != NULL)
 1749                         umtx_pi_disown(pi);
 1750                 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
 1751                         ("blocked queue not empty"));
 1752                 mtx_unlock(&umtx_lock);
 1753                 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
 1754                 umtx_pi_free(pi);
 1755         }
 1756 }
 1757 
 1758 /*
 1759  * Find a PI mutex in hash table.
 1760  */
 1761 static struct umtx_pi *
 1762 umtx_pi_lookup(struct umtx_key *key)
 1763 {
 1764         struct umtxq_chain *uc;
 1765         struct umtx_pi *pi;
 1766 
 1767         uc = umtxq_getchain(key);
 1768         UMTXQ_LOCKED_ASSERT(uc);
 1769 
 1770         TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
 1771                 if (umtx_key_match(&pi->pi_key, key)) {
 1772                         return (pi);
 1773                 }
 1774         }
 1775         return (NULL);
 1776 }
 1777 
 1778 /*
 1779  * Insert a PI mutex into hash table.
 1780  */
 1781 static inline void
 1782 umtx_pi_insert(struct umtx_pi *pi)
 1783 {
 1784         struct umtxq_chain *uc;
 1785 
 1786         uc = umtxq_getchain(&pi->pi_key);
 1787         UMTXQ_LOCKED_ASSERT(uc);
 1788         TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
 1789 }
 1790 
 1791 /*
 1792  * Lock a PI mutex.
 1793  */
 1794 static int
 1795 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 1796     struct _umtx_time *timeout, int try)
 1797 {
 1798         struct abs_timeout timo;
 1799         struct umtx_q *uq;
 1800         struct umtx_pi *pi, *new_pi;
 1801         uint32_t id, old_owner, owner, old;
 1802         int error, rv;
 1803 
 1804         id = td->td_tid;
 1805         uq = td->td_umtxq;
 1806 
 1807         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
 1808             TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
 1809             &uq->uq_key)) != 0)
 1810                 return (error);
 1811 
 1812         if (timeout != NULL)
 1813                 abs_timeout_init2(&timo, timeout);
 1814 
 1815         umtxq_lock(&uq->uq_key);
 1816         pi = umtx_pi_lookup(&uq->uq_key);
 1817         if (pi == NULL) {
 1818                 new_pi = umtx_pi_alloc(M_NOWAIT);
 1819                 if (new_pi == NULL) {
 1820                         umtxq_unlock(&uq->uq_key);
 1821                         new_pi = umtx_pi_alloc(M_WAITOK);
 1822                         umtxq_lock(&uq->uq_key);
 1823                         pi = umtx_pi_lookup(&uq->uq_key);
 1824                         if (pi != NULL) {
 1825                                 umtx_pi_free(new_pi);
 1826                                 new_pi = NULL;
 1827                         }
 1828                 }
 1829                 if (new_pi != NULL) {
 1830                         new_pi->pi_key = uq->uq_key;
 1831                         umtx_pi_insert(new_pi);
 1832                         pi = new_pi;
 1833                 }
 1834         }
 1835         umtx_pi_ref(pi);
 1836         umtxq_unlock(&uq->uq_key);
 1837 
 1838         /*
 1839          * Care must be exercised when dealing with umtx structure.  It
 1840          * can fault on any access.
 1841          */
 1842         for (;;) {
 1843                 /*
 1844                  * Try the uncontested case.  This should be done in userland.
 1845                  */
 1846                 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
 1847                 /* The address was invalid. */
 1848                 if (rv == -1) {
 1849                         error = EFAULT;
 1850                         break;
 1851                 }
 1852 
 1853                 /* The acquire succeeded. */
 1854                 if (owner == UMUTEX_UNOWNED) {
 1855                         error = 0;
 1856                         break;
 1857                 }
 1858 
 1859                 if (owner == UMUTEX_RB_NOTRECOV) {
 1860                         error = ENOTRECOVERABLE;
 1861                         break;
 1862                 }
 1863 
 1864                 /* If no one owns it but it is contested try to acquire it. */
 1865                 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) {
 1866                         old_owner = owner;
 1867                         rv = casueword32(&m->m_owner, owner, &owner,
 1868                             id | UMUTEX_CONTESTED);
 1869                         /* The address was invalid. */
 1870                         if (rv == -1) {
 1871                                 error = EFAULT;
 1872                                 break;
 1873                         }
 1874 
 1875                         if (owner == old_owner) {
 1876                                 umtxq_lock(&uq->uq_key);
 1877                                 umtxq_busy(&uq->uq_key);
 1878                                 error = umtx_pi_claim(pi, td);
 1879                                 umtxq_unbusy(&uq->uq_key);
 1880                                 umtxq_unlock(&uq->uq_key);
 1881                                 if (error != 0) {
 1882                                         /*
 1883                                          * Since we're going to return an
 1884                                          * error, restore the m_owner to its
 1885                                          * previous, unowned state to avoid
 1886                                          * compounding the problem.
 1887                                          */
 1888                                         (void)casuword32(&m->m_owner,
 1889                                             id | UMUTEX_CONTESTED,
 1890                                             old_owner);
 1891                                 }
 1892                                 if (error == 0 &&
 1893                                     old_owner == UMUTEX_RB_OWNERDEAD)
 1894                                         error = EOWNERDEAD;
 1895                                 break;
 1896                         }
 1897 
 1898                         error = umtxq_check_susp(td);
 1899                         if (error != 0)
 1900                                 break;
 1901 
 1902                         /* If this failed the lock has changed, restart. */
 1903                         continue;
 1904                 }
 1905 
 1906                 if ((owner & ~UMUTEX_CONTESTED) == id) {
 1907                         error = EDEADLK;
 1908                         break;
 1909                 }
 1910 
 1911                 if (try != 0) {
 1912                         error = EBUSY;
 1913                         break;
 1914                 }
 1915 
 1916                 /*
 1917                  * If we caught a signal, we have retried and now
 1918                  * exit immediately.
 1919                  */
 1920                 if (error != 0)
 1921                         break;
 1922 
 1923                 umtxq_lock(&uq->uq_key);
 1924                 umtxq_busy(&uq->uq_key);
 1925                 umtxq_unlock(&uq->uq_key);
 1926 
 1927                 /*
 1928                  * Set the contested bit so that a release in user space
 1929                  * knows to use the system call for unlock.  If this fails
 1930                  * either some one else has acquired the lock or it has been
 1931                  * released.
 1932                  */
 1933                 rv = casueword32(&m->m_owner, owner, &old, owner |
 1934                     UMUTEX_CONTESTED);
 1935 
 1936                 /* The address was invalid. */
 1937                 if (rv == -1) {
 1938                         umtxq_unbusy_unlocked(&uq->uq_key);
 1939                         error = EFAULT;
 1940                         break;
 1941                 }
 1942 
 1943                 umtxq_lock(&uq->uq_key);
 1944                 /*
 1945                  * We set the contested bit, sleep. Otherwise the lock changed
 1946                  * and we need to retry or we lost a race to the thread
 1947                  * unlocking the umtx.  Note that the UMUTEX_RB_OWNERDEAD
 1948                  * value for owner is impossible there.
 1949                  */
 1950                 if (old == owner) {
 1951                         error = umtxq_sleep_pi(uq, pi,
 1952                             owner & ~UMUTEX_CONTESTED,
 1953                             "umtxpi", timeout == NULL ? NULL : &timo,
 1954                             (flags & USYNC_PROCESS_SHARED) != 0);
 1955                         if (error != 0)
 1956                                 continue;
 1957                 } else {
 1958                         umtxq_unbusy(&uq->uq_key);
 1959                         umtxq_unlock(&uq->uq_key);
 1960                 }
 1961 
 1962                 error = umtxq_check_susp(td);
 1963                 if (error != 0)
 1964                         break;
 1965         }
 1966 
 1967         umtxq_lock(&uq->uq_key);
 1968         umtx_pi_unref(pi);
 1969         umtxq_unlock(&uq->uq_key);
 1970 
 1971         umtx_key_release(&uq->uq_key);
 1972         return (error);
 1973 }
 1974 
 1975 /*
 1976  * Unlock a PI mutex.
 1977  */
 1978 static int
 1979 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
 1980 {
 1981         struct umtx_key key;
 1982         struct umtx_q *uq_first, *uq_first2, *uq_me;
 1983         struct umtx_pi *pi, *pi2;
 1984         uint32_t id, new_owner, old, owner;
 1985         int count, error, pri;
 1986 
 1987         id = td->td_tid;
 1988         /*
 1989          * Make sure we own this mtx.
 1990          */
 1991         error = fueword32(&m->m_owner, &owner);
 1992         if (error == -1)
 1993                 return (EFAULT);
 1994 
 1995         if ((owner & ~UMUTEX_CONTESTED) != id)
 1996                 return (EPERM);
 1997 
 1998         new_owner = umtx_unlock_val(flags, rb);
 1999 
 2000         /* This should be done in userland */
 2001         if ((owner & UMUTEX_CONTESTED) == 0) {
 2002                 error = casueword32(&m->m_owner, owner, &old, new_owner);
 2003                 if (error == -1)
 2004                         return (EFAULT);
 2005                 if (old == owner)
 2006                         return (0);
 2007                 owner = old;
 2008         }
 2009 
 2010         /* We should only ever be in here for contested locks */
 2011         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
 2012             TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
 2013             &key)) != 0)
 2014                 return (error);
 2015 
 2016         umtxq_lock(&key);
 2017         umtxq_busy(&key);
 2018         count = umtxq_count_pi(&key, &uq_first);
 2019         if (uq_first != NULL) {
 2020                 mtx_lock(&umtx_lock);
 2021                 pi = uq_first->uq_pi_blocked;
 2022                 KASSERT(pi != NULL, ("pi == NULL?"));
 2023                 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
 2024                         mtx_unlock(&umtx_lock);
 2025                         umtxq_unbusy(&key);
 2026                         umtxq_unlock(&key);
 2027                         umtx_key_release(&key);
 2028                         /* userland messed the mutex */
 2029                         return (EPERM);
 2030                 }
 2031                 uq_me = td->td_umtxq;
 2032                 if (pi->pi_owner == td)
 2033                         umtx_pi_disown(pi);
 2034                 /* get highest priority thread which is still sleeping. */
 2035                 uq_first = TAILQ_FIRST(&pi->pi_blocked);
 2036                 while (uq_first != NULL &&
 2037                     (uq_first->uq_flags & UQF_UMTXQ) == 0) {
 2038                         uq_first = TAILQ_NEXT(uq_first, uq_lockq);
 2039                 }
 2040                 pri = PRI_MAX;
 2041                 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
 2042                         uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
 2043                         if (uq_first2 != NULL) {
 2044                                 if (pri > UPRI(uq_first2->uq_thread))
 2045                                         pri = UPRI(uq_first2->uq_thread);
 2046                         }
 2047                 }
 2048                 thread_lock(td);
 2049                 sched_lend_user_prio(td, pri);
 2050                 thread_unlock(td);
 2051                 mtx_unlock(&umtx_lock);
 2052                 if (uq_first)
 2053                         umtxq_signal_thread(uq_first);
 2054         } else {
 2055                 pi = umtx_pi_lookup(&key);
 2056                 /*
 2057                  * A umtx_pi can exist if a signal or timeout removed the
 2058                  * last waiter from the umtxq, but there is still
 2059                  * a thread in do_lock_pi() holding the umtx_pi.
 2060                  */
 2061                 if (pi != NULL) {
 2062                         /*
 2063                          * The umtx_pi can be unowned, such as when a thread
 2064                          * has just entered do_lock_pi(), allocated the
 2065                          * umtx_pi, and unlocked the umtxq.
 2066                          * If the current thread owns it, it must disown it.
 2067                          */
 2068                         mtx_lock(&umtx_lock);
 2069                         if (pi->pi_owner == td)
 2070                                 umtx_pi_disown(pi);
 2071                         mtx_unlock(&umtx_lock);
 2072                 }
 2073         }
 2074         umtxq_unlock(&key);
 2075 
 2076         /*
 2077          * When unlocking the umtx, it must be marked as unowned if
 2078          * there is zero or one thread only waiting for it.
 2079          * Otherwise, it must be marked as contested.
 2080          */
 2081 
 2082         if (count > 1)
 2083                 new_owner |= UMUTEX_CONTESTED;
 2084         error = casueword32(&m->m_owner, owner, &old, new_owner);
 2085 
 2086         umtxq_unbusy_unlocked(&key);
 2087         umtx_key_release(&key);
 2088         if (error == -1)
 2089                 return (EFAULT);
 2090         if (old != owner)
 2091                 return (EINVAL);
 2092         return (0);
 2093 }
 2094 
 2095 /*
 2096  * Lock a PP mutex.
 2097  */
 2098 static int
 2099 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 2100     struct _umtx_time *timeout, int try)
 2101 {
 2102         struct abs_timeout timo;
 2103         struct umtx_q *uq, *uq2;
 2104         struct umtx_pi *pi;
 2105         uint32_t ceiling;
 2106         uint32_t owner, id;
 2107         int error, pri, old_inherited_pri, su, rv;
 2108 
 2109         id = td->td_tid;
 2110         uq = td->td_umtxq;
 2111         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
 2112             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
 2113             &uq->uq_key)) != 0)
 2114                 return (error);
 2115 
 2116         if (timeout != NULL)
 2117                 abs_timeout_init2(&timo, timeout);
 2118 
 2119         su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
 2120         for (;;) {
 2121                 old_inherited_pri = uq->uq_inherited_pri;
 2122                 umtxq_lock(&uq->uq_key);
 2123                 umtxq_busy(&uq->uq_key);
 2124                 umtxq_unlock(&uq->uq_key);
 2125 
 2126                 rv = fueword32(&m->m_ceilings[0], &ceiling);
 2127                 if (rv == -1) {
 2128                         error = EFAULT;
 2129                         goto out;
 2130                 }
 2131                 ceiling = RTP_PRIO_MAX - ceiling;
 2132                 if (ceiling > RTP_PRIO_MAX) {
 2133                         error = EINVAL;
 2134                         goto out;
 2135                 }
 2136 
 2137                 mtx_lock(&umtx_lock);
 2138                 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
 2139                         mtx_unlock(&umtx_lock);
 2140                         error = EINVAL;
 2141                         goto out;
 2142                 }
 2143                 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
 2144                         uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
 2145                         thread_lock(td);
 2146                         if (uq->uq_inherited_pri < UPRI(td))
 2147                                 sched_lend_user_prio(td, uq->uq_inherited_pri);
 2148                         thread_unlock(td);
 2149                 }
 2150                 mtx_unlock(&umtx_lock);
 2151 
 2152                 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
 2153                     id | UMUTEX_CONTESTED);
 2154                 /* The address was invalid. */
 2155                 if (rv == -1) {
 2156                         error = EFAULT;
 2157                         break;
 2158                 }
 2159 
 2160                 if (owner == UMUTEX_CONTESTED) {
 2161                         error = 0;
 2162                         break;
 2163                 } else if (owner == UMUTEX_RB_OWNERDEAD) {
 2164                         rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
 2165                             &owner, id | UMUTEX_CONTESTED);
 2166                         if (rv == -1) {
 2167                                 error = EFAULT;
 2168                                 break;
 2169                         }
 2170                         if (owner == UMUTEX_RB_OWNERDEAD) {
 2171                                 error = EOWNERDEAD; /* success */
 2172                                 break;
 2173                         }
 2174                         error = 0;
 2175                 } else if (owner == UMUTEX_RB_NOTRECOV) {
 2176                         error = ENOTRECOVERABLE;
 2177                         break;
 2178                 }
 2179 
 2180                 if (try != 0) {
 2181                         error = EBUSY;
 2182                         break;
 2183                 }
 2184 
 2185                 /*
 2186                  * If we caught a signal, we have retried and now
 2187                  * exit immediately.
 2188                  */
 2189                 if (error != 0)
 2190                         break;
 2191 
 2192                 umtxq_lock(&uq->uq_key);
 2193                 umtxq_insert(uq);
 2194                 umtxq_unbusy(&uq->uq_key);
 2195                 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
 2196                     NULL : &timo);
 2197                 umtxq_remove(uq);
 2198                 umtxq_unlock(&uq->uq_key);
 2199 
 2200                 mtx_lock(&umtx_lock);
 2201                 uq->uq_inherited_pri = old_inherited_pri;
 2202                 pri = PRI_MAX;
 2203                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
 2204                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
 2205                         if (uq2 != NULL) {
 2206                                 if (pri > UPRI(uq2->uq_thread))
 2207                                         pri = UPRI(uq2->uq_thread);
 2208                         }
 2209                 }
 2210                 if (pri > uq->uq_inherited_pri)
 2211                         pri = uq->uq_inherited_pri;
 2212                 thread_lock(td);
 2213                 sched_lend_user_prio(td, pri);
 2214                 thread_unlock(td);
 2215                 mtx_unlock(&umtx_lock);
 2216         }
 2217 
 2218         if (error != 0 && error != EOWNERDEAD) {
 2219                 mtx_lock(&umtx_lock);
 2220                 uq->uq_inherited_pri = old_inherited_pri;
 2221                 pri = PRI_MAX;
 2222                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
 2223                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
 2224                         if (uq2 != NULL) {
 2225                                 if (pri > UPRI(uq2->uq_thread))
 2226                                         pri = UPRI(uq2->uq_thread);
 2227                         }
 2228                 }
 2229                 if (pri > uq->uq_inherited_pri)
 2230                         pri = uq->uq_inherited_pri;
 2231                 thread_lock(td);
 2232                 sched_lend_user_prio(td, pri);
 2233                 thread_unlock(td);
 2234                 mtx_unlock(&umtx_lock);
 2235         }
 2236 
 2237 out:
 2238         umtxq_unbusy_unlocked(&uq->uq_key);
 2239         umtx_key_release(&uq->uq_key);
 2240         return (error);
 2241 }
 2242 
 2243 /*
 2244  * Unlock a PP mutex.
 2245  */
 2246 static int
 2247 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
 2248 {
 2249         struct umtx_key key;
 2250         struct umtx_q *uq, *uq2;
 2251         struct umtx_pi *pi;
 2252         uint32_t id, owner, rceiling;
 2253         int error, pri, new_inherited_pri, su;
 2254 
 2255         id = td->td_tid;
 2256         uq = td->td_umtxq;
 2257         su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
 2258 
 2259         /*
 2260          * Make sure we own this mtx.
 2261          */
 2262         error = fueword32(&m->m_owner, &owner);
 2263         if (error == -1)
 2264                 return (EFAULT);
 2265 
 2266         if ((owner & ~UMUTEX_CONTESTED) != id)
 2267                 return (EPERM);
 2268 
 2269         error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
 2270         if (error != 0)
 2271                 return (error);
 2272 
 2273         if (rceiling == -1)
 2274                 new_inherited_pri = PRI_MAX;
 2275         else {
 2276                 rceiling = RTP_PRIO_MAX - rceiling;
 2277                 if (rceiling > RTP_PRIO_MAX)
 2278                         return (EINVAL);
 2279                 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
 2280         }
 2281 
 2282         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
 2283             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
 2284             &key)) != 0)
 2285                 return (error);
 2286         umtxq_lock(&key);
 2287         umtxq_busy(&key);
 2288         umtxq_unlock(&key);
 2289         /*
 2290          * For priority protected mutex, always set unlocked state
 2291          * to UMUTEX_CONTESTED, so that userland always enters kernel
 2292          * to lock the mutex, it is necessary because thread priority
 2293          * has to be adjusted for such mutex.
 2294          */
 2295         error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
 2296             UMUTEX_CONTESTED);
 2297 
 2298         umtxq_lock(&key);
 2299         if (error == 0)
 2300                 umtxq_signal(&key, 1);
 2301         umtxq_unbusy(&key);
 2302         umtxq_unlock(&key);
 2303 
 2304         if (error == -1)
 2305                 error = EFAULT;
 2306         else {
 2307                 mtx_lock(&umtx_lock);
 2308                 if (su != 0)
 2309                         uq->uq_inherited_pri = new_inherited_pri;
 2310                 pri = PRI_MAX;
 2311                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
 2312                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
 2313                         if (uq2 != NULL) {
 2314                                 if (pri > UPRI(uq2->uq_thread))
 2315                                         pri = UPRI(uq2->uq_thread);
 2316                         }
 2317                 }
 2318                 if (pri > uq->uq_inherited_pri)
 2319                         pri = uq->uq_inherited_pri;
 2320                 thread_lock(td);
 2321                 sched_lend_user_prio(td, pri);
 2322                 thread_unlock(td);
 2323                 mtx_unlock(&umtx_lock);
 2324         }
 2325         umtx_key_release(&key);
 2326         return (error);
 2327 }
 2328 
 2329 static int
 2330 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
 2331     uint32_t *old_ceiling)
 2332 {
 2333         struct umtx_q *uq;
 2334         uint32_t flags, id, owner, save_ceiling;
 2335         int error, rv, rv1;
 2336 
 2337         error = fueword32(&m->m_flags, &flags);
 2338         if (error == -1)
 2339                 return (EFAULT);
 2340         if ((flags & UMUTEX_PRIO_PROTECT) == 0)
 2341                 return (EINVAL);
 2342         if (ceiling > RTP_PRIO_MAX)
 2343                 return (EINVAL);
 2344         id = td->td_tid;
 2345         uq = td->td_umtxq;
 2346         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
 2347             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
 2348             &uq->uq_key)) != 0)
 2349                 return (error);
 2350         for (;;) {
 2351                 umtxq_lock(&uq->uq_key);
 2352                 umtxq_busy(&uq->uq_key);
 2353                 umtxq_unlock(&uq->uq_key);
 2354 
 2355                 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
 2356                 if (rv == -1) {
 2357                         error = EFAULT;
 2358                         break;
 2359                 }
 2360 
 2361                 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
 2362                     id | UMUTEX_CONTESTED);
 2363                 if (rv == -1) {
 2364                         error = EFAULT;
 2365                         break;
 2366                 }
 2367 
 2368                 if (owner == UMUTEX_CONTESTED) {
 2369                         rv = suword32(&m->m_ceilings[0], ceiling);
 2370                         rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
 2371                         error = (rv == 0 && rv1 == 0) ? 0: EFAULT;
 2372                         break;
 2373                 }
 2374 
 2375                 if ((owner & ~UMUTEX_CONTESTED) == id) {
 2376                         rv = suword32(&m->m_ceilings[0], ceiling);
 2377                         error = rv == 0 ? 0 : EFAULT;
 2378                         break;
 2379                 }
 2380 
 2381                 if (owner == UMUTEX_RB_OWNERDEAD) {
 2382                         error = EOWNERDEAD;
 2383                         break;
 2384                 } else if (owner == UMUTEX_RB_NOTRECOV) {
 2385                         error = ENOTRECOVERABLE;
 2386                         break;
 2387                 }
 2388 
 2389                 /*
 2390                  * If we caught a signal, we have retried and now
 2391                  * exit immediately.
 2392                  */
 2393                 if (error != 0)
 2394                         break;
 2395 
 2396                 /*
 2397                  * We set the contested bit, sleep. Otherwise the lock changed
 2398                  * and we need to retry or we lost a race to the thread
 2399                  * unlocking the umtx.
 2400                  */
 2401                 umtxq_lock(&uq->uq_key);
 2402                 umtxq_insert(uq);
 2403                 umtxq_unbusy(&uq->uq_key);
 2404                 error = umtxq_sleep(uq, "umtxpp", NULL);
 2405                 umtxq_remove(uq);
 2406                 umtxq_unlock(&uq->uq_key);
 2407         }
 2408         umtxq_lock(&uq->uq_key);
 2409         if (error == 0)
 2410                 umtxq_signal(&uq->uq_key, INT_MAX);
 2411         umtxq_unbusy(&uq->uq_key);
 2412         umtxq_unlock(&uq->uq_key);
 2413         umtx_key_release(&uq->uq_key);
 2414         if (error == 0 && old_ceiling != NULL) {
 2415                 rv = suword32(old_ceiling, save_ceiling);
 2416                 error = rv == 0 ? 0 : EFAULT;
 2417         }
 2418         return (error);
 2419 }
 2420 
 2421 /*
 2422  * Lock a userland POSIX mutex.
 2423  */
 2424 static int
 2425 do_lock_umutex(struct thread *td, struct umutex *m,
 2426     struct _umtx_time *timeout, int mode)
 2427 {
 2428         uint32_t flags;
 2429         int error;
 2430 
 2431         error = fueword32(&m->m_flags, &flags);
 2432         if (error == -1)
 2433                 return (EFAULT);
 2434 
 2435         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
 2436         case 0:
 2437                 error = do_lock_normal(td, m, flags, timeout, mode);
 2438                 break;
 2439         case UMUTEX_PRIO_INHERIT:
 2440                 error = do_lock_pi(td, m, flags, timeout, mode);
 2441                 break;
 2442         case UMUTEX_PRIO_PROTECT:
 2443                 error = do_lock_pp(td, m, flags, timeout, mode);
 2444                 break;
 2445         default:
 2446                 return (EINVAL);
 2447         }
 2448         if (timeout == NULL) {
 2449                 if (error == EINTR && mode != _UMUTEX_WAIT)
 2450                         error = ERESTART;
 2451         } else {
 2452                 /* Timed-locking is not restarted. */
 2453                 if (error == ERESTART)
 2454                         error = EINTR;
 2455         }
 2456         return (error);
 2457 }
 2458 
 2459 /*
 2460  * Unlock a userland POSIX mutex.
 2461  */
 2462 static int
 2463 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb)
 2464 {
 2465         uint32_t flags;
 2466         int error;
 2467 
 2468         error = fueword32(&m->m_flags, &flags);
 2469         if (error == -1)
 2470                 return (EFAULT);
 2471 
 2472         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
 2473         case 0:
 2474                 return (do_unlock_normal(td, m, flags, rb));
 2475         case UMUTEX_PRIO_INHERIT:
 2476                 return (do_unlock_pi(td, m, flags, rb));
 2477         case UMUTEX_PRIO_PROTECT:
 2478                 return (do_unlock_pp(td, m, flags, rb));
 2479         }
 2480 
 2481         return (EINVAL);
 2482 }
 2483 
 2484 static int
 2485 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
 2486     struct timespec *timeout, u_long wflags)
 2487 {
 2488         struct abs_timeout timo;
 2489         struct umtx_q *uq;
 2490         uint32_t flags, clockid, hasw;
 2491         int error;
 2492 
 2493         uq = td->td_umtxq;
 2494         error = fueword32(&cv->c_flags, &flags);
 2495         if (error == -1)
 2496                 return (EFAULT);
 2497         error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
 2498         if (error != 0)
 2499                 return (error);
 2500 
 2501         if ((wflags & CVWAIT_CLOCKID) != 0) {
 2502                 error = fueword32(&cv->c_clockid, &clockid);
 2503                 if (error == -1) {
 2504                         umtx_key_release(&uq->uq_key);
 2505                         return (EFAULT);
 2506                 }
 2507                 if (clockid < CLOCK_REALTIME ||
 2508                     clockid >= CLOCK_THREAD_CPUTIME_ID) {
 2509                         /* hmm, only HW clock id will work. */
 2510                         umtx_key_release(&uq->uq_key);
 2511                         return (EINVAL);
 2512                 }
 2513         } else {
 2514                 clockid = CLOCK_REALTIME;
 2515         }
 2516 
 2517         umtxq_lock(&uq->uq_key);
 2518         umtxq_busy(&uq->uq_key);
 2519         umtxq_insert(uq);
 2520         umtxq_unlock(&uq->uq_key);
 2521 
 2522         /*
 2523          * Set c_has_waiters to 1 before releasing user mutex, also
 2524          * don't modify cache line when unnecessary.
 2525          */
 2526         error = fueword32(&cv->c_has_waiters, &hasw);
 2527         if (error == 0 && hasw == 0)
 2528                 suword32(&cv->c_has_waiters, 1);
 2529 
 2530         umtxq_unbusy_unlocked(&uq->uq_key);
 2531 
 2532         error = do_unlock_umutex(td, m, false);
 2533 
 2534         if (timeout != NULL)
 2535                 abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0,
 2536                     timeout);
 2537 
 2538         umtxq_lock(&uq->uq_key);
 2539         if (error == 0) {
 2540                 error = umtxq_sleep(uq, "ucond", timeout == NULL ?
 2541                     NULL : &timo);
 2542         }
 2543 
 2544         if ((uq->uq_flags & UQF_UMTXQ) == 0)
 2545                 error = 0;
 2546         else {
 2547                 /*
 2548                  * This must be timeout,interrupted by signal or
 2549                  * surprious wakeup, clear c_has_waiter flag when
 2550                  * necessary.
 2551                  */
 2552                 umtxq_busy(&uq->uq_key);
 2553                 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
 2554                         int oldlen = uq->uq_cur_queue->length;
 2555                         umtxq_remove(uq);
 2556                         if (oldlen == 1) {
 2557                                 umtxq_unlock(&uq->uq_key);
 2558                                 suword32(&cv->c_has_waiters, 0);
 2559                                 umtxq_lock(&uq->uq_key);
 2560                         }
 2561                 }
 2562                 umtxq_unbusy(&uq->uq_key);
 2563                 if (error == ERESTART)
 2564                         error = EINTR;
 2565         }
 2566 
 2567         umtxq_unlock(&uq->uq_key);
 2568         umtx_key_release(&uq->uq_key);
 2569         return (error);
 2570 }
 2571 
 2572 /*
 2573  * Signal a userland condition variable.
 2574  */
 2575 static int
 2576 do_cv_signal(struct thread *td, struct ucond *cv)
 2577 {
 2578         struct umtx_key key;
 2579         int error, cnt, nwake;
 2580         uint32_t flags;
 2581 
 2582         error = fueword32(&cv->c_flags, &flags);
 2583         if (error == -1)
 2584                 return (EFAULT);
 2585         if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
 2586                 return (error);
 2587         umtxq_lock(&key);
 2588         umtxq_busy(&key);
 2589         cnt = umtxq_count(&key);
 2590         nwake = umtxq_signal(&key, 1);
 2591         if (cnt <= nwake) {
 2592                 umtxq_unlock(&key);
 2593                 error = suword32(&cv->c_has_waiters, 0);
 2594                 if (error == -1)
 2595                         error = EFAULT;
 2596                 umtxq_lock(&key);
 2597         }
 2598         umtxq_unbusy(&key);
 2599         umtxq_unlock(&key);
 2600         umtx_key_release(&key);
 2601         return (error);
 2602 }
 2603 
 2604 static int
 2605 do_cv_broadcast(struct thread *td, struct ucond *cv)
 2606 {
 2607         struct umtx_key key;
 2608         int error;
 2609         uint32_t flags;
 2610 
 2611         error = fueword32(&cv->c_flags, &flags);
 2612         if (error == -1)
 2613                 return (EFAULT);
 2614         if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
 2615                 return (error);
 2616 
 2617         umtxq_lock(&key);
 2618         umtxq_busy(&key);
 2619         umtxq_signal(&key, INT_MAX);
 2620         umtxq_unlock(&key);
 2621 
 2622         error = suword32(&cv->c_has_waiters, 0);
 2623         if (error == -1)
 2624                 error = EFAULT;
 2625 
 2626         umtxq_unbusy_unlocked(&key);
 2627 
 2628         umtx_key_release(&key);
 2629         return (error);
 2630 }
 2631 
 2632 static int
 2633 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
 2634     struct _umtx_time *timeout)
 2635 {
 2636         struct abs_timeout timo;
 2637         struct umtx_q *uq;
 2638         uint32_t flags, wrflags;
 2639         int32_t state, oldstate;
 2640         int32_t blocked_readers;
 2641         int error, error1, rv;
 2642 
 2643         uq = td->td_umtxq;
 2644         error = fueword32(&rwlock->rw_flags, &flags);
 2645         if (error == -1)
 2646                 return (EFAULT);
 2647         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 2648         if (error != 0)
 2649                 return (error);
 2650 
 2651         if (timeout != NULL)
 2652                 abs_timeout_init2(&timo, timeout);
 2653 
 2654         wrflags = URWLOCK_WRITE_OWNER;
 2655         if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
 2656                 wrflags |= URWLOCK_WRITE_WAITERS;
 2657 
 2658         for (;;) {
 2659                 rv = fueword32(&rwlock->rw_state, &state);
 2660                 if (rv == -1) {
 2661                         umtx_key_release(&uq->uq_key);
 2662                         return (EFAULT);
 2663                 }
 2664 
 2665                 /* try to lock it */
 2666                 while (!(state & wrflags)) {
 2667                         if (__predict_false(URWLOCK_READER_COUNT(state) ==
 2668                             URWLOCK_MAX_READERS)) {
 2669                                 umtx_key_release(&uq->uq_key);
 2670                                 return (EAGAIN);
 2671                         }
 2672                         rv = casueword32(&rwlock->rw_state, state,
 2673                             &oldstate, state + 1);
 2674                         if (rv == -1) {
 2675                                 umtx_key_release(&uq->uq_key);
 2676                                 return (EFAULT);
 2677                         }
 2678                         if (oldstate == state) {
 2679                                 umtx_key_release(&uq->uq_key);
 2680                                 return (0);
 2681                         }
 2682                         error = umtxq_check_susp(td);
 2683                         if (error != 0)
 2684                                 break;
 2685                         state = oldstate;
 2686                 }
 2687 
 2688                 if (error)
 2689                         break;
 2690 
 2691                 /* grab monitor lock */
 2692                 umtxq_lock(&uq->uq_key);
 2693                 umtxq_busy(&uq->uq_key);
 2694                 umtxq_unlock(&uq->uq_key);
 2695 
 2696                 /*
 2697                  * re-read the state, in case it changed between the try-lock above
 2698                  * and the check below
 2699                  */
 2700                 rv = fueword32(&rwlock->rw_state, &state);
 2701                 if (rv == -1)
 2702                         error = EFAULT;
 2703 
 2704                 /* set read contention bit */
 2705                 while (error == 0 && (state & wrflags) &&
 2706                     !(state & URWLOCK_READ_WAITERS)) {
 2707                         rv = casueword32(&rwlock->rw_state, state,
 2708                             &oldstate, state | URWLOCK_READ_WAITERS);
 2709                         if (rv == -1) {
 2710                                 error = EFAULT;
 2711                                 break;
 2712                         }
 2713                         if (oldstate == state)
 2714                                 goto sleep;
 2715                         state = oldstate;
 2716                         error = umtxq_check_susp(td);
 2717                         if (error != 0)
 2718                                 break;
 2719                 }
 2720                 if (error != 0) {
 2721                         umtxq_unbusy_unlocked(&uq->uq_key);
 2722                         break;
 2723                 }
 2724 
 2725                 /* state is changed while setting flags, restart */
 2726                 if (!(state & wrflags)) {
 2727                         umtxq_unbusy_unlocked(&uq->uq_key);
 2728                         error = umtxq_check_susp(td);
 2729                         if (error != 0)
 2730                                 break;
 2731                         continue;
 2732                 }
 2733 
 2734 sleep:
 2735                 /*
 2736                  * Contention bit is set, before sleeping, increase
 2737                  * read waiter count.
 2738                  */
 2739                 rv = fueword32(&rwlock->rw_blocked_readers,
 2740                     &blocked_readers);
 2741                 if (rv == -1) {
 2742                         umtxq_unbusy_unlocked(&uq->uq_key);
 2743                         error = EFAULT;
 2744                         break;
 2745                 }
 2746                 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
 2747 
 2748                 while (state & wrflags) {
 2749                         umtxq_lock(&uq->uq_key);
 2750                         umtxq_insert(uq);
 2751                         umtxq_unbusy(&uq->uq_key);
 2752 
 2753                         error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
 2754                             NULL : &timo);
 2755 
 2756                         umtxq_busy(&uq->uq_key);
 2757                         umtxq_remove(uq);
 2758                         umtxq_unlock(&uq->uq_key);
 2759                         if (error)
 2760                                 break;
 2761                         rv = fueword32(&rwlock->rw_state, &state);
 2762                         if (rv == -1) {
 2763                                 error = EFAULT;
 2764                                 break;
 2765                         }
 2766                 }
 2767 
 2768                 /* decrease read waiter count, and may clear read contention bit */
 2769                 rv = fueword32(&rwlock->rw_blocked_readers,
 2770                     &blocked_readers);
 2771                 if (rv == -1) {
 2772                         umtxq_unbusy_unlocked(&uq->uq_key);
 2773                         error = EFAULT;
 2774                         break;
 2775                 }
 2776                 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
 2777                 if (blocked_readers == 1) {
 2778                         rv = fueword32(&rwlock->rw_state, &state);
 2779                         if (rv == -1) {
 2780                                 umtxq_unbusy_unlocked(&uq->uq_key);
 2781                                 error = EFAULT;
 2782                                 break;
 2783                         }
 2784                         for (;;) {
 2785                                 rv = casueword32(&rwlock->rw_state, state,
 2786                                     &oldstate, state & ~URWLOCK_READ_WAITERS);
 2787                                 if (rv == -1) {
 2788                                         error = EFAULT;
 2789                                         break;
 2790                                 }
 2791                                 if (oldstate == state)
 2792                                         break;
 2793                                 state = oldstate;
 2794                                 error1 = umtxq_check_susp(td);
 2795                                 if (error1 != 0) {
 2796                                         if (error == 0)
 2797                                                 error = error1;
 2798                                         break;
 2799                                 }
 2800                         }
 2801                 }
 2802 
 2803                 umtxq_unbusy_unlocked(&uq->uq_key);
 2804                 if (error != 0)
 2805                         break;
 2806         }
 2807         umtx_key_release(&uq->uq_key);
 2808         if (error == ERESTART)
 2809                 error = EINTR;
 2810         return (error);
 2811 }
 2812 
 2813 static int
 2814 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout)
 2815 {
 2816         struct abs_timeout timo;
 2817         struct umtx_q *uq;
 2818         uint32_t flags;
 2819         int32_t state, oldstate;
 2820         int32_t blocked_writers;
 2821         int32_t blocked_readers;
 2822         int error, error1, rv;
 2823 
 2824         uq = td->td_umtxq;
 2825         error = fueword32(&rwlock->rw_flags, &flags);
 2826         if (error == -1)
 2827                 return (EFAULT);
 2828         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 2829         if (error != 0)
 2830                 return (error);
 2831 
 2832         if (timeout != NULL)
 2833                 abs_timeout_init2(&timo, timeout);
 2834 
 2835         blocked_readers = 0;
 2836         for (;;) {
 2837                 rv = fueword32(&rwlock->rw_state, &state);
 2838                 if (rv == -1) {
 2839                         umtx_key_release(&uq->uq_key);
 2840                         return (EFAULT);
 2841                 }
 2842                 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
 2843                     URWLOCK_READER_COUNT(state) == 0) {
 2844                         rv = casueword32(&rwlock->rw_state, state,
 2845                             &oldstate, state | URWLOCK_WRITE_OWNER);
 2846                         if (rv == -1) {
 2847                                 umtx_key_release(&uq->uq_key);
 2848                                 return (EFAULT);
 2849                         }
 2850                         if (oldstate == state) {
 2851                                 umtx_key_release(&uq->uq_key);
 2852                                 return (0);
 2853                         }
 2854                         state = oldstate;
 2855                         error = umtxq_check_susp(td);
 2856                         if (error != 0)
 2857                                 break;
 2858                 }
 2859 
 2860                 if (error) {
 2861                         if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
 2862                             blocked_readers != 0) {
 2863                                 umtxq_lock(&uq->uq_key);
 2864                                 umtxq_busy(&uq->uq_key);
 2865                                 umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
 2866                                 umtxq_unbusy(&uq->uq_key);
 2867                                 umtxq_unlock(&uq->uq_key);
 2868                         }
 2869 
 2870                         break;
 2871                 }
 2872 
 2873                 /* grab monitor lock */
 2874                 umtxq_lock(&uq->uq_key);
 2875                 umtxq_busy(&uq->uq_key);
 2876                 umtxq_unlock(&uq->uq_key);
 2877 
 2878                 /*
 2879                  * Re-read the state, in case it changed between the
 2880                  * try-lock above and the check below.
 2881                  */
 2882                 rv = fueword32(&rwlock->rw_state, &state);
 2883                 if (rv == -1)
 2884                         error = EFAULT;
 2885 
 2886                 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
 2887                     URWLOCK_READER_COUNT(state) != 0) &&
 2888                     (state & URWLOCK_WRITE_WAITERS) == 0) {
 2889                         rv = casueword32(&rwlock->rw_state, state,
 2890                             &oldstate, state | URWLOCK_WRITE_WAITERS);
 2891                         if (rv == -1) {
 2892                                 error = EFAULT;
 2893                                 break;
 2894                         }
 2895                         if (oldstate == state)
 2896                                 goto sleep;
 2897                         state = oldstate;
 2898                         error = umtxq_check_susp(td);
 2899                         if (error != 0)
 2900                                 break;
 2901                 }
 2902                 if (error != 0) {
 2903                         umtxq_unbusy_unlocked(&uq->uq_key);
 2904                         break;
 2905                 }
 2906 
 2907                 if ((state & URWLOCK_WRITE_OWNER) == 0 &&
 2908                     URWLOCK_READER_COUNT(state) == 0) {
 2909                         umtxq_unbusy_unlocked(&uq->uq_key);
 2910                         error = umtxq_check_susp(td);
 2911                         if (error != 0)
 2912                                 break;
 2913                         continue;
 2914                 }
 2915 sleep:
 2916                 rv = fueword32(&rwlock->rw_blocked_writers,
 2917                     &blocked_writers);
 2918                 if (rv == -1) {
 2919                         umtxq_unbusy_unlocked(&uq->uq_key);
 2920                         error = EFAULT;
 2921                         break;
 2922                 }
 2923                 suword32(&rwlock->rw_blocked_writers, blocked_writers + 1);
 2924 
 2925                 while ((state & URWLOCK_WRITE_OWNER) ||
 2926                     URWLOCK_READER_COUNT(state) != 0) {
 2927                         umtxq_lock(&uq->uq_key);
 2928                         umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
 2929                         umtxq_unbusy(&uq->uq_key);
 2930 
 2931                         error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
 2932                             NULL : &timo);
 2933 
 2934                         umtxq_busy(&uq->uq_key);
 2935                         umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
 2936                         umtxq_unlock(&uq->uq_key);
 2937                         if (error)
 2938                                 break;
 2939                         rv = fueword32(&rwlock->rw_state, &state);
 2940                         if (rv == -1) {
 2941                                 error = EFAULT;
 2942                                 break;
 2943                         }
 2944                 }
 2945 
 2946                 rv = fueword32(&rwlock->rw_blocked_writers,
 2947                     &blocked_writers);
 2948                 if (rv == -1) {
 2949                         umtxq_unbusy_unlocked(&uq->uq_key);
 2950                         error = EFAULT;
 2951                         break;
 2952                 }
 2953                 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
 2954                 if (blocked_writers == 1) {
 2955                         rv = fueword32(&rwlock->rw_state, &state);
 2956                         if (rv == -1) {
 2957                                 umtxq_unbusy_unlocked(&uq->uq_key);
 2958                                 error = EFAULT;
 2959                                 break;
 2960                         }
 2961                         for (;;) {
 2962                                 rv = casueword32(&rwlock->rw_state, state,
 2963                                     &oldstate, state & ~URWLOCK_WRITE_WAITERS);
 2964                                 if (rv == -1) {
 2965                                         error = EFAULT;
 2966                                         break;
 2967                                 }
 2968                                 if (oldstate == state)
 2969                                         break;
 2970                                 state = oldstate;
 2971                                 error1 = umtxq_check_susp(td);
 2972                                 /*
 2973                                  * We are leaving the URWLOCK_WRITE_WAITERS
 2974                                  * behind, but this should not harm the
 2975                                  * correctness.
 2976                                  */
 2977                                 if (error1 != 0) {
 2978                                         if (error == 0)
 2979                                                 error = error1;
 2980                                         break;
 2981                                 }
 2982                         }
 2983                         rv = fueword32(&rwlock->rw_blocked_readers,
 2984                             &blocked_readers);
 2985                         if (rv == -1) {
 2986                                 umtxq_unbusy_unlocked(&uq->uq_key);
 2987                                 error = EFAULT;
 2988                                 break;
 2989                         }
 2990                 } else
 2991                         blocked_readers = 0;
 2992 
 2993                 umtxq_unbusy_unlocked(&uq->uq_key);
 2994         }
 2995 
 2996         umtx_key_release(&uq->uq_key);
 2997         if (error == ERESTART)
 2998                 error = EINTR;
 2999         return (error);
 3000 }
 3001 
 3002 static int
 3003 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
 3004 {
 3005         struct umtx_q *uq;
 3006         uint32_t flags;
 3007         int32_t state, oldstate;
 3008         int error, rv, q, count;
 3009 
 3010         uq = td->td_umtxq;
 3011         error = fueword32(&rwlock->rw_flags, &flags);
 3012         if (error == -1)
 3013                 return (EFAULT);
 3014         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 3015         if (error != 0)
 3016                 return (error);
 3017 
 3018         error = fueword32(&rwlock->rw_state, &state);
 3019         if (error == -1) {
 3020                 error = EFAULT;
 3021                 goto out;
 3022         }
 3023         if (state & URWLOCK_WRITE_OWNER) {
 3024                 for (;;) {
 3025                         rv = casueword32(&rwlock->rw_state, state,
 3026                             &oldstate, state & ~URWLOCK_WRITE_OWNER);
 3027                         if (rv == -1) {
 3028                                 error = EFAULT;
 3029                                 goto out;
 3030                         }
 3031                         if (oldstate != state) {
 3032                                 state = oldstate;
 3033                                 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
 3034                                         error = EPERM;
 3035                                         goto out;
 3036                                 }
 3037                                 error = umtxq_check_susp(td);
 3038                                 if (error != 0)
 3039                                         goto out;
 3040                         } else
 3041                                 break;
 3042                 }
 3043         } else if (URWLOCK_READER_COUNT(state) != 0) {
 3044                 for (;;) {
 3045                         rv = casueword32(&rwlock->rw_state, state,
 3046                             &oldstate, state - 1);
 3047                         if (rv == -1) {
 3048                                 error = EFAULT;
 3049                                 goto out;
 3050                         }
 3051                         if (oldstate != state) {
 3052                                 state = oldstate;
 3053                                 if (URWLOCK_READER_COUNT(oldstate) == 0) {
 3054                                         error = EPERM;
 3055                                         goto out;
 3056                                 }
 3057                                 error = umtxq_check_susp(td);
 3058                                 if (error != 0)
 3059                                         goto out;
 3060                         } else
 3061                                 break;
 3062                 }
 3063         } else {
 3064                 error = EPERM;
 3065                 goto out;
 3066         }
 3067 
 3068         count = 0;
 3069 
 3070         if (!(flags & URWLOCK_PREFER_READER)) {
 3071                 if (state & URWLOCK_WRITE_WAITERS) {
 3072                         count = 1;
 3073                         q = UMTX_EXCLUSIVE_QUEUE;
 3074                 } else if (state & URWLOCK_READ_WAITERS) {
 3075                         count = INT_MAX;
 3076                         q = UMTX_SHARED_QUEUE;
 3077                 }
 3078         } else {
 3079                 if (state & URWLOCK_READ_WAITERS) {
 3080                         count = INT_MAX;
 3081                         q = UMTX_SHARED_QUEUE;
 3082                 } else if (state & URWLOCK_WRITE_WAITERS) {
 3083                         count = 1;
 3084                         q = UMTX_EXCLUSIVE_QUEUE;
 3085                 }
 3086         }
 3087 
 3088         if (count) {
 3089                 umtxq_lock(&uq->uq_key);
 3090                 umtxq_busy(&uq->uq_key);
 3091                 umtxq_signal_queue(&uq->uq_key, count, q);
 3092                 umtxq_unbusy(&uq->uq_key);
 3093                 umtxq_unlock(&uq->uq_key);
 3094         }
 3095 out:
 3096         umtx_key_release(&uq->uq_key);
 3097         return (error);
 3098 }
 3099 
 3100 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
 3101 static int
 3102 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
 3103 {
 3104         struct abs_timeout timo;
 3105         struct umtx_q *uq;
 3106         uint32_t flags, count, count1;
 3107         int error, rv;
 3108 
 3109         uq = td->td_umtxq;
 3110         error = fueword32(&sem->_flags, &flags);
 3111         if (error == -1)
 3112                 return (EFAULT);
 3113         error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
 3114         if (error != 0)
 3115                 return (error);
 3116 
 3117         if (timeout != NULL)
 3118                 abs_timeout_init2(&timo, timeout);
 3119 
 3120         umtxq_lock(&uq->uq_key);
 3121         umtxq_busy(&uq->uq_key);
 3122         umtxq_insert(uq);
 3123         umtxq_unlock(&uq->uq_key);
 3124         rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
 3125         if (rv == 0)
 3126                 rv = fueword32(&sem->_count, &count);
 3127         if (rv == -1 || count != 0) {
 3128                 umtxq_lock(&uq->uq_key);
 3129                 umtxq_unbusy(&uq->uq_key);
 3130                 umtxq_remove(uq);
 3131                 umtxq_unlock(&uq->uq_key);
 3132                 umtx_key_release(&uq->uq_key);
 3133                 return (rv == -1 ? EFAULT : 0);
 3134         }
 3135         umtxq_lock(&uq->uq_key);
 3136         umtxq_unbusy(&uq->uq_key);
 3137 
 3138         error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
 3139 
 3140         if ((uq->uq_flags & UQF_UMTXQ) == 0)
 3141                 error = 0;
 3142         else {
 3143                 umtxq_remove(uq);
 3144                 /* A relative timeout cannot be restarted. */
 3145                 if (error == ERESTART && timeout != NULL &&
 3146                     (timeout->_flags & UMTX_ABSTIME) == 0)
 3147                         error = EINTR;
 3148         }
 3149         umtxq_unlock(&uq->uq_key);
 3150         umtx_key_release(&uq->uq_key);
 3151         return (error);
 3152 }
 3153 
 3154 /*
 3155  * Signal a userland semaphore.
 3156  */
 3157 static int
 3158 do_sem_wake(struct thread *td, struct _usem *sem)
 3159 {
 3160         struct umtx_key key;
 3161         int error, cnt;
 3162         uint32_t flags;
 3163 
 3164         error = fueword32(&sem->_flags, &flags);
 3165         if (error == -1)
 3166                 return (EFAULT);
 3167         if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
 3168                 return (error);
 3169         umtxq_lock(&key);
 3170         umtxq_busy(&key);
 3171         cnt = umtxq_count(&key);
 3172         if (cnt > 0) {
 3173                 /*
 3174                  * Check if count is greater than 0, this means the memory is
 3175                  * still being referenced by user code, so we can safely
 3176                  * update _has_waiters flag.
 3177                  */
 3178                 if (cnt == 1) {
 3179                         umtxq_unlock(&key);
 3180                         error = suword32(&sem->_has_waiters, 0);
 3181                         umtxq_lock(&key);
 3182                         if (error == -1)
 3183                                 error = EFAULT;
 3184                 }
 3185                 umtxq_signal(&key, 1);
 3186         }
 3187         umtxq_unbusy(&key);
 3188         umtxq_unlock(&key);
 3189         umtx_key_release(&key);
 3190         return (error);
 3191 }
 3192 #endif
 3193 
 3194 static int
 3195 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
 3196 {
 3197         struct abs_timeout timo;
 3198         struct umtx_q *uq;
 3199         uint32_t count, flags;
 3200         int error, rv;
 3201 
 3202         uq = td->td_umtxq;
 3203         flags = fuword32(&sem->_flags);
 3204         error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
 3205         if (error != 0)
 3206                 return (error);
 3207 
 3208         if (timeout != NULL)
 3209                 abs_timeout_init2(&timo, timeout);
 3210 
 3211         umtxq_lock(&uq->uq_key);
 3212         umtxq_busy(&uq->uq_key);
 3213         umtxq_insert(uq);
 3214         umtxq_unlock(&uq->uq_key);
 3215         rv = fueword32(&sem->_count, &count);
 3216         if (rv == -1) {
 3217                 umtxq_lock(&uq->uq_key);
 3218                 umtxq_unbusy(&uq->uq_key);
 3219                 umtxq_remove(uq);
 3220                 umtxq_unlock(&uq->uq_key);
 3221                 umtx_key_release(&uq->uq_key);
 3222                 return (EFAULT);
 3223         }
 3224         for (;;) {
 3225                 if (USEM_COUNT(count) != 0) {
 3226                         umtxq_lock(&uq->uq_key);
 3227                         umtxq_unbusy(&uq->uq_key);
 3228                         umtxq_remove(uq);
 3229                         umtxq_unlock(&uq->uq_key);
 3230                         umtx_key_release(&uq->uq_key);
 3231                         return (0);
 3232                 }
 3233                 if (count == USEM_HAS_WAITERS)
 3234                         break;
 3235                 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
 3236                 if (rv == -1) {
 3237                         umtxq_lock(&uq->uq_key);
 3238                         umtxq_unbusy(&uq->uq_key);
 3239                         umtxq_remove(uq);
 3240                         umtxq_unlock(&uq->uq_key);
 3241                         umtx_key_release(&uq->uq_key);
 3242                         return (EFAULT);
 3243                 }
 3244                 if (count == 0)
 3245                         break;
 3246         }
 3247         umtxq_lock(&uq->uq_key);
 3248         umtxq_unbusy(&uq->uq_key);
 3249 
 3250         error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
 3251 
 3252         if ((uq->uq_flags & UQF_UMTXQ) == 0)
 3253                 error = 0;
 3254         else {
 3255                 umtxq_remove(uq);
 3256                 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
 3257                         /* A relative timeout cannot be restarted. */
 3258                         if (error == ERESTART)
 3259                                 error = EINTR;
 3260                         if (error == EINTR) {
 3261                                 abs_timeout_update(&timo);
 3262                                 timeout->_timeout = timo.end;
 3263                                 timespecsub(&timeout->_timeout, &timo.cur);
 3264                         }
 3265                 }
 3266         }
 3267         umtxq_unlock(&uq->uq_key);
 3268         umtx_key_release(&uq->uq_key);
 3269         return (error);
 3270 }
 3271 
 3272 /*
 3273  * Signal a userland semaphore.
 3274  */
 3275 static int
 3276 do_sem2_wake(struct thread *td, struct _usem2 *sem)
 3277 {
 3278         struct umtx_key key;
 3279         int error, cnt, rv;
 3280         uint32_t count, flags;
 3281 
 3282         rv = fueword32(&sem->_flags, &flags);
 3283         if (rv == -1)
 3284                 return (EFAULT);
 3285         if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
 3286                 return (error);
 3287         umtxq_lock(&key);
 3288         umtxq_busy(&key);
 3289         cnt = umtxq_count(&key);
 3290         if (cnt > 0) {
 3291                 /*
 3292                  * If this was the last sleeping thread, clear the waiters
 3293                  * flag in _count.
 3294                  */
 3295                 if (cnt == 1) {
 3296                         umtxq_unlock(&key);
 3297                         rv = fueword32(&sem->_count, &count);
 3298                         while (rv != -1 && count & USEM_HAS_WAITERS)
 3299                                 rv = casueword32(&sem->_count, count, &count,
 3300                                     count & ~USEM_HAS_WAITERS);
 3301                         if (rv == -1)
 3302                                 error = EFAULT;
 3303                         umtxq_lock(&key);
 3304                 }
 3305 
 3306                 umtxq_signal(&key, 1);
 3307         }
 3308         umtxq_unbusy(&key);
 3309         umtxq_unlock(&key);
 3310         umtx_key_release(&key);
 3311         return (error);
 3312 }
 3313 
 3314 inline int
 3315 umtx_copyin_timeout(const void *addr, struct timespec *tsp)
 3316 {
 3317         int error;
 3318 
 3319         error = copyin(addr, tsp, sizeof(struct timespec));
 3320         if (error == 0) {
 3321                 if (tsp->tv_sec < 0 ||
 3322                     tsp->tv_nsec >= 1000000000 ||
 3323                     tsp->tv_nsec < 0)
 3324                         error = EINVAL;
 3325         }
 3326         return (error);
 3327 }
 3328 
 3329 static inline int
 3330 umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp)
 3331 {
 3332         int error;
 3333 
 3334         if (size <= sizeof(struct timespec)) {
 3335                 tp->_clockid = CLOCK_REALTIME;
 3336                 tp->_flags = 0;
 3337                 error = copyin(addr, &tp->_timeout, sizeof(struct timespec));
 3338         } else
 3339                 error = copyin(addr, tp, sizeof(struct _umtx_time));
 3340         if (error != 0)
 3341                 return (error);
 3342         if (tp->_timeout.tv_sec < 0 ||
 3343             tp->_timeout.tv_nsec >= 1000000000 || tp->_timeout.tv_nsec < 0)
 3344                 return (EINVAL);
 3345         return (0);
 3346 }
 3347 
 3348 static int
 3349 __umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap)
 3350 {
 3351 
 3352         return (EOPNOTSUPP);
 3353 }
 3354 
 3355 static int
 3356 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
 3357 {
 3358         struct _umtx_time timeout, *tm_p;
 3359         int error;
 3360 
 3361         if (uap->uaddr2 == NULL)
 3362                 tm_p = NULL;
 3363         else {
 3364                 error = umtx_copyin_umtx_time(
 3365                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3366                 if (error != 0)
 3367                         return (error);
 3368                 tm_p = &timeout;
 3369         }
 3370         return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0));
 3371 }
 3372 
 3373 static int
 3374 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
 3375 {
 3376         struct _umtx_time timeout, *tm_p;
 3377         int error;
 3378 
 3379         if (uap->uaddr2 == NULL)
 3380                 tm_p = NULL;
 3381         else {
 3382                 error = umtx_copyin_umtx_time(
 3383                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3384                 if (error != 0)
 3385                         return (error);
 3386                 tm_p = &timeout;
 3387         }
 3388         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
 3389 }
 3390 
 3391 static int
 3392 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
 3393 {
 3394         struct _umtx_time *tm_p, timeout;
 3395         int error;
 3396 
 3397         if (uap->uaddr2 == NULL)
 3398                 tm_p = NULL;
 3399         else {
 3400                 error = umtx_copyin_umtx_time(
 3401                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3402                 if (error != 0)
 3403                         return (error);
 3404                 tm_p = &timeout;
 3405         }
 3406         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
 3407 }
 3408 
 3409 static int
 3410 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
 3411 {
 3412 
 3413         return (kern_umtx_wake(td, uap->obj, uap->val, 0));
 3414 }
 3415 
 3416 #define BATCH_SIZE      128
 3417 static int
 3418 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap)
 3419 {
 3420         char *uaddrs[BATCH_SIZE], **upp;
 3421         int count, error, i, pos, tocopy;
 3422 
 3423         upp = (char **)uap->obj;
 3424         error = 0;
 3425         for (count = uap->val, pos = 0; count > 0; count -= tocopy,
 3426             pos += tocopy) {
 3427                 tocopy = MIN(count, BATCH_SIZE);
 3428                 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *));
 3429                 if (error != 0)
 3430                         break;
 3431                 for (i = 0; i < tocopy; ++i)
 3432                         kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
 3433                 maybe_yield();
 3434         }
 3435         return (error);
 3436 }
 3437 
 3438 static int
 3439 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
 3440 {
 3441 
 3442         return (kern_umtx_wake(td, uap->obj, uap->val, 1));
 3443 }
 3444 
 3445 static int
 3446 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
 3447 {
 3448         struct _umtx_time *tm_p, timeout;
 3449         int error;
 3450 
 3451         /* Allow a null timespec (wait forever). */
 3452         if (uap->uaddr2 == NULL)
 3453                 tm_p = NULL;
 3454         else {
 3455                 error = umtx_copyin_umtx_time(
 3456                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3457                 if (error != 0)
 3458                         return (error);
 3459                 tm_p = &timeout;
 3460         }
 3461         return (do_lock_umutex(td, uap->obj, tm_p, 0));
 3462 }
 3463 
 3464 static int
 3465 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
 3466 {
 3467 
 3468         return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
 3469 }
 3470 
 3471 static int
 3472 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
 3473 {
 3474         struct _umtx_time *tm_p, timeout;
 3475         int error;
 3476 
 3477         /* Allow a null timespec (wait forever). */
 3478         if (uap->uaddr2 == NULL)
 3479                 tm_p = NULL;
 3480         else {
 3481                 error = umtx_copyin_umtx_time(
 3482                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3483                 if (error != 0)
 3484                         return (error);
 3485                 tm_p = &timeout;
 3486         }
 3487         return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
 3488 }
 3489 
 3490 static int
 3491 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
 3492 {
 3493 
 3494         return (do_wake_umutex(td, uap->obj));
 3495 }
 3496 
 3497 static int
 3498 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
 3499 {
 3500 
 3501         return (do_unlock_umutex(td, uap->obj, false));
 3502 }
 3503 
 3504 static int
 3505 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
 3506 {
 3507 
 3508         return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
 3509 }
 3510 
 3511 static int
 3512 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
 3513 {
 3514         struct timespec *ts, timeout;
 3515         int error;
 3516 
 3517         /* Allow a null timespec (wait forever). */
 3518         if (uap->uaddr2 == NULL)
 3519                 ts = NULL;
 3520         else {
 3521                 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
 3522                 if (error != 0)
 3523                         return (error);
 3524                 ts = &timeout;
 3525         }
 3526         return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
 3527 }
 3528 
 3529 static int
 3530 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
 3531 {
 3532 
 3533         return (do_cv_signal(td, uap->obj));
 3534 }
 3535 
 3536 static int
 3537 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
 3538 {
 3539 
 3540         return (do_cv_broadcast(td, uap->obj));
 3541 }
 3542 
 3543 static int
 3544 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
 3545 {
 3546         struct _umtx_time timeout;
 3547         int error;
 3548 
 3549         /* Allow a null timespec (wait forever). */
 3550         if (uap->uaddr2 == NULL) {
 3551                 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
 3552         } else {
 3553                 error = umtx_copyin_umtx_time(uap->uaddr2,
 3554                    (size_t)uap->uaddr1, &timeout);
 3555                 if (error != 0)
 3556                         return (error);
 3557                 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
 3558         }
 3559         return (error);
 3560 }
 3561 
 3562 static int
 3563 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
 3564 {
 3565         struct _umtx_time timeout;
 3566         int error;
 3567 
 3568         /* Allow a null timespec (wait forever). */
 3569         if (uap->uaddr2 == NULL) {
 3570                 error = do_rw_wrlock(td, uap->obj, 0);
 3571         } else {
 3572                 error = umtx_copyin_umtx_time(uap->uaddr2,
 3573                    (size_t)uap->uaddr1, &timeout);
 3574                 if (error != 0)
 3575                         return (error);
 3576 
 3577                 error = do_rw_wrlock(td, uap->obj, &timeout);
 3578         }
 3579         return (error);
 3580 }
 3581 
 3582 static int
 3583 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
 3584 {
 3585 
 3586         return (do_rw_unlock(td, uap->obj));
 3587 }
 3588 
 3589 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
 3590 static int
 3591 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap)
 3592 {
 3593         struct _umtx_time *tm_p, timeout;
 3594         int error;
 3595 
 3596         /* Allow a null timespec (wait forever). */
 3597         if (uap->uaddr2 == NULL)
 3598                 tm_p = NULL;
 3599         else {
 3600                 error = umtx_copyin_umtx_time(
 3601                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
 3602                 if (error != 0)
 3603                         return (error);
 3604                 tm_p = &timeout;
 3605         }
 3606         return (do_sem_wait(td, uap->obj, tm_p));
 3607 }
 3608 
 3609 static int
 3610 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap)
 3611 {
 3612 
 3613         return (do_sem_wake(td, uap->obj));
 3614 }
 3615 #endif
 3616 
 3617 static int
 3618 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap)
 3619 {
 3620 
 3621         return (do_wake2_umutex(td, uap->obj, uap->val));
 3622 }
 3623 
 3624 static int
 3625 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap)
 3626 {
 3627         struct _umtx_time *tm_p, timeout;
 3628         size_t uasize;
 3629         int error;
 3630 
 3631         /* Allow a null timespec (wait forever). */
 3632         if (uap->uaddr2 == NULL) {
 3633                 uasize = 0;
 3634                 tm_p = NULL;
 3635         } else {
 3636                 uasize = (size_t)uap->uaddr1;
 3637                 error = umtx_copyin_umtx_time(uap->uaddr2, uasize, &timeout);
 3638                 if (error != 0)
 3639                         return (error);
 3640                 tm_p = &timeout;
 3641         }
 3642         error = do_sem2_wait(td, uap->obj, tm_p);
 3643         if (error == EINTR && uap->uaddr2 != NULL &&
 3644             (timeout._flags & UMTX_ABSTIME) == 0 &&
 3645             uasize >= sizeof(struct _umtx_time) + sizeof(struct timespec)) {
 3646                 error = copyout(&timeout._timeout,
 3647                     (struct _umtx_time *)uap->uaddr2 + 1,
 3648                     sizeof(struct timespec));
 3649                 if (error == 0) {
 3650                         error = EINTR;
 3651                 }
 3652         }
 3653 
 3654         return (error);
 3655 }
 3656 
 3657 static int
 3658 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap)
 3659 {
 3660 
 3661         return (do_sem2_wake(td, uap->obj));
 3662 }
 3663 
 3664 #define USHM_OBJ_UMTX(o)                                                \
 3665     ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
 3666 
 3667 #define USHMF_REG_LINKED        0x0001
 3668 #define USHMF_OBJ_LINKED        0x0002
 3669 struct umtx_shm_reg {
 3670         TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link;
 3671         LIST_ENTRY(umtx_shm_reg) ushm_obj_link;
 3672         struct umtx_key         ushm_key;
 3673         struct ucred            *ushm_cred;
 3674         struct shmfd            *ushm_obj;
 3675         u_int                   ushm_refcnt;
 3676         u_int                   ushm_flags;
 3677 };
 3678 
 3679 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg);
 3680 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg);
 3681 
 3682 static uma_zone_t umtx_shm_reg_zone;
 3683 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS];
 3684 static struct mtx umtx_shm_lock;
 3685 static struct umtx_shm_reg_head umtx_shm_reg_delfree =
 3686     TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree);
 3687 
 3688 static void umtx_shm_free_reg(struct umtx_shm_reg *reg);
 3689 
 3690 static void
 3691 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused)
 3692 {
 3693         struct umtx_shm_reg_head d;
 3694         struct umtx_shm_reg *reg, *reg1;
 3695 
 3696         TAILQ_INIT(&d);
 3697         mtx_lock(&umtx_shm_lock);
 3698         TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link);
 3699         mtx_unlock(&umtx_shm_lock);
 3700         TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) {
 3701                 TAILQ_REMOVE(&d, reg, ushm_reg_link);
 3702                 umtx_shm_free_reg(reg);
 3703         }
 3704 }
 3705 
 3706 static struct task umtx_shm_reg_delfree_task =
 3707     TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL);
 3708 
 3709 static struct umtx_shm_reg *
 3710 umtx_shm_find_reg_locked(const struct umtx_key *key)
 3711 {
 3712         struct umtx_shm_reg *reg;
 3713         struct umtx_shm_reg_head *reg_head;
 3714 
 3715         KASSERT(key->shared, ("umtx_p_find_rg: private key"));
 3716         mtx_assert(&umtx_shm_lock, MA_OWNED);
 3717         reg_head = &umtx_shm_registry[key->hash];
 3718         TAILQ_FOREACH(reg, reg_head, ushm_reg_link) {
 3719                 KASSERT(reg->ushm_key.shared,
 3720                     ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
 3721                 if (reg->ushm_key.info.shared.object ==
 3722                     key->info.shared.object &&
 3723                     reg->ushm_key.info.shared.offset ==
 3724                     key->info.shared.offset) {
 3725                         KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
 3726                         KASSERT(reg->ushm_refcnt > 0,
 3727                             ("reg %p refcnt 0 onlist", reg));
 3728                         KASSERT((reg->ushm_flags & USHMF_REG_LINKED) != 0,
 3729                             ("reg %p not linked", reg));
 3730                         reg->ushm_refcnt++;
 3731                         return (reg);
 3732                 }
 3733         }
 3734         return (NULL);
 3735 }
 3736 
 3737 static struct umtx_shm_reg *
 3738 umtx_shm_find_reg(const struct umtx_key *key)
 3739 {
 3740         struct umtx_shm_reg *reg;
 3741 
 3742         mtx_lock(&umtx_shm_lock);
 3743         reg = umtx_shm_find_reg_locked(key);
 3744         mtx_unlock(&umtx_shm_lock);
 3745         return (reg);
 3746 }
 3747 
 3748 static void
 3749 umtx_shm_free_reg(struct umtx_shm_reg *reg)
 3750 {
 3751 
 3752         chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
 3753         crfree(reg->ushm_cred);
 3754         shm_drop(reg->ushm_obj);
 3755         uma_zfree(umtx_shm_reg_zone, reg);
 3756 }
 3757 
 3758 static bool
 3759 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool force)
 3760 {
 3761         bool res;
 3762 
 3763         mtx_assert(&umtx_shm_lock, MA_OWNED);
 3764         KASSERT(reg->ushm_refcnt > 0, ("ushm_reg %p refcnt 0", reg));
 3765         reg->ushm_refcnt--;
 3766         res = reg->ushm_refcnt == 0;
 3767         if (res || force) {
 3768                 if ((reg->ushm_flags & USHMF_REG_LINKED) != 0) {
 3769                         TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash],
 3770                             reg, ushm_reg_link);
 3771                         reg->ushm_flags &= ~USHMF_REG_LINKED;
 3772                 }
 3773                 if ((reg->ushm_flags & USHMF_OBJ_LINKED) != 0) {
 3774                         LIST_REMOVE(reg, ushm_obj_link);
 3775                         reg->ushm_flags &= ~USHMF_OBJ_LINKED;
 3776                 }
 3777         }
 3778         return (res);
 3779 }
 3780 
 3781 static void
 3782 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool force)
 3783 {
 3784         vm_object_t object;
 3785         bool dofree;
 3786 
 3787         if (force) {
 3788                 object = reg->ushm_obj->shm_object;
 3789                 VM_OBJECT_WLOCK(object);
 3790                 object->flags |= OBJ_UMTXDEAD;
 3791                 VM_OBJECT_WUNLOCK(object);
 3792         }
 3793         mtx_lock(&umtx_shm_lock);
 3794         dofree = umtx_shm_unref_reg_locked(reg, force);
 3795         mtx_unlock(&umtx_shm_lock);
 3796         if (dofree)
 3797                 umtx_shm_free_reg(reg);
 3798 }
 3799 
 3800 void
 3801 umtx_shm_object_init(vm_object_t object)
 3802 {
 3803 
 3804         LIST_INIT(USHM_OBJ_UMTX(object));
 3805 }
 3806 
 3807 void
 3808 umtx_shm_object_terminated(vm_object_t object)
 3809 {
 3810         struct umtx_shm_reg *reg, *reg1;
 3811         bool dofree;
 3812 
 3813         dofree = false;
 3814         mtx_lock(&umtx_shm_lock);
 3815         LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) {
 3816                 if (umtx_shm_unref_reg_locked(reg, true)) {
 3817                         TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg,
 3818                             ushm_reg_link);
 3819                         dofree = true;
 3820                 }
 3821         }
 3822         mtx_unlock(&umtx_shm_lock);
 3823         if (dofree)
 3824                 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task);
 3825 }
 3826 
 3827 static int
 3828 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
 3829     struct umtx_shm_reg **res)
 3830 {
 3831         struct umtx_shm_reg *reg, *reg1;
 3832         struct ucred *cred;
 3833         int error;
 3834 
 3835         reg = umtx_shm_find_reg(key);
 3836         if (reg != NULL) {
 3837                 *res = reg;
 3838                 return (0);
 3839         }
 3840         cred = td->td_ucred;
 3841         if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
 3842                 return (ENOMEM);
 3843         reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
 3844         reg->ushm_refcnt = 1;
 3845         bcopy(key, &reg->ushm_key, sizeof(*key));
 3846         reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR);
 3847         reg->ushm_cred = crhold(cred);
 3848         error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
 3849         if (error != 0) {
 3850                 umtx_shm_free_reg(reg);
 3851                 return (error);
 3852         }
 3853         mtx_lock(&umtx_shm_lock);
 3854         reg1 = umtx_shm_find_reg_locked(key);
 3855         if (reg1 != NULL) {
 3856                 mtx_unlock(&umtx_shm_lock);
 3857                 umtx_shm_free_reg(reg);
 3858                 *res = reg1;
 3859                 return (0);
 3860         }
 3861         reg->ushm_refcnt++;
 3862         TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
 3863         LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
 3864             ushm_obj_link);
 3865         reg->ushm_flags = USHMF_REG_LINKED | USHMF_OBJ_LINKED;
 3866         mtx_unlock(&umtx_shm_lock);
 3867         *res = reg;
 3868         return (0);
 3869 }
 3870 
 3871 static int
 3872 umtx_shm_alive(struct thread *td, void *addr)
 3873 {
 3874         vm_map_t map;
 3875         vm_map_entry_t entry;
 3876         vm_object_t object;
 3877         vm_pindex_t pindex;
 3878         vm_prot_t prot;
 3879         int res, ret;
 3880         boolean_t wired;
 3881 
 3882         map = &td->td_proc->p_vmspace->vm_map;
 3883         res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry,
 3884             &object, &pindex, &prot, &wired);
 3885         if (res != KERN_SUCCESS)
 3886                 return (EFAULT);
 3887         if (object == NULL)
 3888                 ret = EINVAL;
 3889         else
 3890                 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
 3891         vm_map_lookup_done(map, entry);
 3892         return (ret);
 3893 }
 3894 
 3895 static void
 3896 umtx_shm_init(void)
 3897 {
 3898         int i;
 3899 
 3900         umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg),
 3901             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 3902         mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF);
 3903         for (i = 0; i < nitems(umtx_shm_registry); i++)
 3904                 TAILQ_INIT(&umtx_shm_registry[i]);
 3905 }
 3906 
 3907 static int
 3908 umtx_shm(struct thread *td, void *addr, u_int flags)
 3909 {
 3910         struct umtx_key key;
 3911         struct umtx_shm_reg *reg;
 3912         struct file *fp;
 3913         int error, fd;
 3914 
 3915         if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP |
 3916             UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1)
 3917                 return (EINVAL);
 3918         if ((flags & UMTX_SHM_ALIVE) != 0)
 3919                 return (umtx_shm_alive(td, addr));
 3920         error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key);
 3921         if (error != 0)
 3922                 return (error);
 3923         KASSERT(key.shared == 1, ("non-shared key"));
 3924         if ((flags & UMTX_SHM_CREAT) != 0) {
 3925                 error = umtx_shm_create_reg(td, &key, &reg);
 3926         } else {
 3927                 reg = umtx_shm_find_reg(&key);
 3928                 if (reg == NULL)
 3929                         error = ESRCH;
 3930         }
 3931         umtx_key_release(&key);
 3932         if (error != 0)
 3933                 return (error);
 3934         KASSERT(reg != NULL, ("no reg"));
 3935         if ((flags & UMTX_SHM_DESTROY) != 0) {
 3936                 umtx_shm_unref_reg(reg, true);
 3937         } else {
 3938 #if 0
 3939 #ifdef MAC
 3940                 error = mac_posixshm_check_open(td->td_ucred,
 3941                     reg->ushm_obj, FFLAGS(O_RDWR));
 3942                 if (error == 0)
 3943 #endif
 3944                         error = shm_access(reg->ushm_obj, td->td_ucred,
 3945                             FFLAGS(O_RDWR));
 3946                 if (error == 0)
 3947 #endif
 3948                         error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL);
 3949                 if (error == 0) {
 3950                         shm_hold(reg->ushm_obj);
 3951                         finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
 3952                             &shm_ops);
 3953                         td->td_retval[0] = fd;
 3954                         fdrop(fp, td);
 3955                 }
 3956         }
 3957         umtx_shm_unref_reg(reg, false);
 3958         return (error);
 3959 }
 3960 
 3961 static int
 3962 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap)
 3963 {
 3964 
 3965         return (umtx_shm(td, uap->uaddr1, uap->val));
 3966 }
 3967 
 3968 static int
 3969 umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp)
 3970 {
 3971 
 3972         td->td_rb_list = rbp->robust_list_offset;
 3973         td->td_rbp_list = rbp->robust_priv_list_offset;
 3974         td->td_rb_inact = rbp->robust_inact_offset;
 3975         return (0);
 3976 }
 3977 
 3978 static int
 3979 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap)
 3980 {
 3981         struct umtx_robust_lists_params rb;
 3982         int error;
 3983 
 3984         if (uap->val > sizeof(rb))
 3985                 return (EINVAL);
 3986         bzero(&rb, sizeof(rb));
 3987         error = copyin(uap->uaddr1, &rb, uap->val);
 3988         if (error != 0)
 3989                 return (error);
 3990         return (umtx_robust_lists(td, &rb));
 3991 }
 3992 
 3993 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
 3994 
 3995 static const _umtx_op_func op_table[] = {
 3996         [UMTX_OP_RESERVED0]     = __umtx_op_unimpl,
 3997         [UMTX_OP_RESERVED1]     = __umtx_op_unimpl,
 3998         [UMTX_OP_WAIT]          = __umtx_op_wait,
 3999         [UMTX_OP_WAKE]          = __umtx_op_wake,
 4000         [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
 4001         [UMTX_OP_MUTEX_LOCK]    = __umtx_op_lock_umutex,
 4002         [UMTX_OP_MUTEX_UNLOCK]  = __umtx_op_unlock_umutex,
 4003         [UMTX_OP_SET_CEILING]   = __umtx_op_set_ceiling,
 4004         [UMTX_OP_CV_WAIT]       = __umtx_op_cv_wait,
 4005         [UMTX_OP_CV_SIGNAL]     = __umtx_op_cv_signal,
 4006         [UMTX_OP_CV_BROADCAST]  = __umtx_op_cv_broadcast,
 4007         [UMTX_OP_WAIT_UINT]     = __umtx_op_wait_uint,
 4008         [UMTX_OP_RW_RDLOCK]     = __umtx_op_rw_rdlock,
 4009         [UMTX_OP_RW_WRLOCK]     = __umtx_op_rw_wrlock,
 4010         [UMTX_OP_RW_UNLOCK]     = __umtx_op_rw_unlock,
 4011         [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private,
 4012         [UMTX_OP_WAKE_PRIVATE]  = __umtx_op_wake_private,
 4013         [UMTX_OP_MUTEX_WAIT]    = __umtx_op_wait_umutex,
 4014         [UMTX_OP_MUTEX_WAKE]    = __umtx_op_wake_umutex,
 4015 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
 4016         [UMTX_OP_SEM_WAIT]      = __umtx_op_sem_wait,
 4017         [UMTX_OP_SEM_WAKE]      = __umtx_op_sem_wake,
 4018 #else
 4019         [UMTX_OP_SEM_WAIT]      = __umtx_op_unimpl,
 4020         [UMTX_OP_SEM_WAKE]      = __umtx_op_unimpl,
 4021 #endif
 4022         [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private,
 4023         [UMTX_OP_MUTEX_WAKE2]   = __umtx_op_wake2_umutex,
 4024         [UMTX_OP_SEM2_WAIT]     = __umtx_op_sem2_wait,
 4025         [UMTX_OP_SEM2_WAKE]     = __umtx_op_sem2_wake,
 4026         [UMTX_OP_SHM]           = __umtx_op_shm,
 4027         [UMTX_OP_ROBUST_LISTS]  = __umtx_op_robust_lists,
 4028 };
 4029 
 4030 int
 4031 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
 4032 {
 4033 
 4034         if ((unsigned)uap->op < nitems(op_table))
 4035                 return (*op_table[uap->op])(td, uap);
 4036         return (EINVAL);
 4037 }
 4038 
 4039 #ifdef COMPAT_FREEBSD32
 4040 
 4041 struct timespec32 {
 4042         int32_t tv_sec;
 4043         int32_t tv_nsec;
 4044 };
 4045 
 4046 struct umtx_time32 {
 4047         struct  timespec32      timeout;
 4048         uint32_t                flags;
 4049         uint32_t                clockid;
 4050 };
 4051 
 4052 static inline int
 4053 umtx_copyin_timeout32(void *addr, struct timespec *tsp)
 4054 {
 4055         struct timespec32 ts32;
 4056         int error;
 4057 
 4058         error = copyin(addr, &ts32, sizeof(struct timespec32));
 4059         if (error == 0) {
 4060                 if (ts32.tv_sec < 0 ||
 4061                     ts32.tv_nsec >= 1000000000 ||
 4062                     ts32.tv_nsec < 0)
 4063                         error = EINVAL;
 4064                 else {
 4065                         tsp->tv_sec = ts32.tv_sec;
 4066                         tsp->tv_nsec = ts32.tv_nsec;
 4067                 }
 4068         }
 4069         return (error);
 4070 }
 4071 
 4072 static inline int
 4073 umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp)
 4074 {
 4075         struct umtx_time32 t32;
 4076         int error;
 4077 
 4078         t32.clockid = CLOCK_REALTIME;
 4079         t32.flags   = 0;
 4080         if (size <= sizeof(struct timespec32))
 4081                 error = copyin(addr, &t32.timeout, sizeof(struct timespec32));
 4082         else
 4083                 error = copyin(addr, &t32, sizeof(struct umtx_time32));
 4084         if (error != 0)
 4085                 return (error);
 4086         if (t32.timeout.tv_sec < 0 ||
 4087             t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0)
 4088                 return (EINVAL);
 4089         tp->_timeout.tv_sec = t32.timeout.tv_sec;
 4090         tp->_timeout.tv_nsec = t32.timeout.tv_nsec;
 4091         tp->_flags = t32.flags;
 4092         tp->_clockid = t32.clockid;
 4093         return (0);
 4094 }
 4095 
 4096 static int
 4097 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
 4098 {
 4099         struct _umtx_time *tm_p, timeout;
 4100         int error;
 4101 
 4102         if (uap->uaddr2 == NULL)
 4103                 tm_p = NULL;
 4104         else {
 4105                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4106                         (size_t)uap->uaddr1, &timeout);
 4107                 if (error != 0)
 4108                         return (error);
 4109                 tm_p = &timeout;
 4110         }
 4111         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
 4112 }
 4113 
 4114 static int
 4115 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
 4116 {
 4117         struct _umtx_time *tm_p, timeout;
 4118         int error;
 4119 
 4120         /* Allow a null timespec (wait forever). */
 4121         if (uap->uaddr2 == NULL)
 4122                 tm_p = NULL;
 4123         else {
 4124                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4125                             (size_t)uap->uaddr1, &timeout);
 4126                 if (error != 0)
 4127                         return (error);
 4128                 tm_p = &timeout;
 4129         }
 4130         return (do_lock_umutex(td, uap->obj, tm_p, 0));
 4131 }
 4132 
 4133 static int
 4134 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
 4135 {
 4136         struct _umtx_time *tm_p, timeout;
 4137         int error;
 4138 
 4139         /* Allow a null timespec (wait forever). */
 4140         if (uap->uaddr2 == NULL)
 4141                 tm_p = NULL;
 4142         else {
 4143                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4144                     (size_t)uap->uaddr1, &timeout);
 4145                 if (error != 0)
 4146                         return (error);
 4147                 tm_p = &timeout;
 4148         }
 4149         return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
 4150 }
 4151 
 4152 static int
 4153 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
 4154 {
 4155         struct timespec *ts, timeout;
 4156         int error;
 4157 
 4158         /* Allow a null timespec (wait forever). */
 4159         if (uap->uaddr2 == NULL)
 4160                 ts = NULL;
 4161         else {
 4162                 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
 4163                 if (error != 0)
 4164                         return (error);
 4165                 ts = &timeout;
 4166         }
 4167         return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
 4168 }
 4169 
 4170 static int
 4171 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
 4172 {
 4173         struct _umtx_time timeout;
 4174         int error;
 4175 
 4176         /* Allow a null timespec (wait forever). */
 4177         if (uap->uaddr2 == NULL) {
 4178                 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
 4179         } else {
 4180                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4181                     (size_t)uap->uaddr1, &timeout);
 4182                 if (error != 0)
 4183                         return (error);
 4184                 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
 4185         }
 4186         return (error);
 4187 }
 4188 
 4189 static int
 4190 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
 4191 {
 4192         struct _umtx_time timeout;
 4193         int error;
 4194 
 4195         /* Allow a null timespec (wait forever). */
 4196         if (uap->uaddr2 == NULL) {
 4197                 error = do_rw_wrlock(td, uap->obj, 0);
 4198         } else {
 4199                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4200                     (size_t)uap->uaddr1, &timeout);
 4201                 if (error != 0)
 4202                         return (error);
 4203                 error = do_rw_wrlock(td, uap->obj, &timeout);
 4204         }
 4205         return (error);
 4206 }
 4207 
 4208 static int
 4209 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
 4210 {
 4211         struct _umtx_time *tm_p, timeout;
 4212         int error;
 4213 
 4214         if (uap->uaddr2 == NULL)
 4215                 tm_p = NULL;
 4216         else {
 4217                 error = umtx_copyin_umtx_time32(
 4218                     uap->uaddr2, (size_t)uap->uaddr1,&timeout);
 4219                 if (error != 0)
 4220                         return (error);
 4221                 tm_p = &timeout;
 4222         }
 4223         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
 4224 }
 4225 
 4226 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
 4227 static int
 4228 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
 4229 {
 4230         struct _umtx_time *tm_p, timeout;
 4231         int error;
 4232 
 4233         /* Allow a null timespec (wait forever). */
 4234         if (uap->uaddr2 == NULL)
 4235                 tm_p = NULL;
 4236         else {
 4237                 error = umtx_copyin_umtx_time32(uap->uaddr2,
 4238                     (size_t)uap->uaddr1, &timeout);
 4239                 if (error != 0)
 4240                         return (error);
 4241                 tm_p = &timeout;
 4242         }
 4243         return (do_sem_wait(td, uap->obj, tm_p));
 4244 }
 4245 #endif
 4246 
 4247 static int
 4248 __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
 4249 {
 4250         struct _umtx_time *tm_p, timeout;
 4251         size_t uasize;
 4252         int error;
 4253 
 4254         /* Allow a null timespec (wait forever). */
 4255         if (uap->uaddr2 == NULL) {
 4256                 uasize = 0;
 4257                 tm_p = NULL;
 4258         } else {
 4259                 uasize = (size_t)uap->uaddr1;
 4260                 error = umtx_copyin_umtx_time32(uap->uaddr2, uasize, &timeout);
 4261                 if (error != 0)
 4262                         return (error);
 4263                 tm_p = &timeout;
 4264         }
 4265         error = do_sem2_wait(td, uap->obj, tm_p);
 4266         if (error == EINTR && uap->uaddr2 != NULL &&
 4267             (timeout._flags & UMTX_ABSTIME) == 0 &&
 4268             uasize >= sizeof(struct umtx_time32) + sizeof(struct timespec32)) {
 4269                 struct timespec32 remain32 = {
 4270                         .tv_sec = timeout._timeout.tv_sec,
 4271                         .tv_nsec = timeout._timeout.tv_nsec
 4272                 };
 4273                 error = copyout(&remain32,
 4274                     (struct umtx_time32 *)uap->uaddr2 + 1,
 4275                     sizeof(struct timespec32));
 4276                 if (error == 0) {
 4277                         error = EINTR;
 4278                 }
 4279         }
 4280 
 4281         return (error);
 4282 }
 4283 
 4284 static int
 4285 __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap)
 4286 {
 4287         uint32_t uaddrs[BATCH_SIZE], **upp;
 4288         int count, error, i, pos, tocopy;
 4289 
 4290         upp = (uint32_t **)uap->obj;
 4291         error = 0;
 4292         for (count = uap->val, pos = 0; count > 0; count -= tocopy,
 4293             pos += tocopy) {
 4294                 tocopy = MIN(count, BATCH_SIZE);
 4295                 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t));
 4296                 if (error != 0)
 4297                         break;
 4298                 for (i = 0; i < tocopy; ++i)
 4299                         kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i],
 4300                             INT_MAX, 1);
 4301                 maybe_yield();
 4302         }
 4303         return (error);
 4304 }
 4305 
 4306 struct umtx_robust_lists_params_compat32 {
 4307         uint32_t        robust_list_offset;
 4308         uint32_t        robust_priv_list_offset;
 4309         uint32_t        robust_inact_offset;
 4310 };
 4311 
 4312 static int
 4313 __umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap)
 4314 {
 4315         struct umtx_robust_lists_params rb;
 4316         struct umtx_robust_lists_params_compat32 rb32;
 4317         int error;
 4318 
 4319         if (uap->val > sizeof(rb32))
 4320                 return (EINVAL);
 4321         bzero(&rb, sizeof(rb));
 4322         bzero(&rb32, sizeof(rb32));
 4323         error = copyin(uap->uaddr1, &rb32, uap->val);
 4324         if (error != 0)
 4325                 return (error);
 4326         rb.robust_list_offset = rb32.robust_list_offset;
 4327         rb.robust_priv_list_offset = rb32.robust_priv_list_offset;
 4328         rb.robust_inact_offset = rb32.robust_inact_offset;
 4329         return (umtx_robust_lists(td, &rb));
 4330 }
 4331 
 4332 static const _umtx_op_func op_table_compat32[] = {
 4333         [UMTX_OP_RESERVED0]     = __umtx_op_unimpl,
 4334         [UMTX_OP_RESERVED1]     = __umtx_op_unimpl,
 4335         [UMTX_OP_WAIT]          = __umtx_op_wait_compat32,
 4336         [UMTX_OP_WAKE]          = __umtx_op_wake,
 4337         [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
 4338         [UMTX_OP_MUTEX_LOCK]    = __umtx_op_lock_umutex_compat32,
 4339         [UMTX_OP_MUTEX_UNLOCK]  = __umtx_op_unlock_umutex,
 4340         [UMTX_OP_SET_CEILING]   = __umtx_op_set_ceiling,
 4341         [UMTX_OP_CV_WAIT]       = __umtx_op_cv_wait_compat32,
 4342         [UMTX_OP_CV_SIGNAL]     = __umtx_op_cv_signal,
 4343         [UMTX_OP_CV_BROADCAST]  = __umtx_op_cv_broadcast,
 4344         [UMTX_OP_WAIT_UINT]     = __umtx_op_wait_compat32,
 4345         [UMTX_OP_RW_RDLOCK]     = __umtx_op_rw_rdlock_compat32,
 4346         [UMTX_OP_RW_WRLOCK]     = __umtx_op_rw_wrlock_compat32,
 4347         [UMTX_OP_RW_UNLOCK]     = __umtx_op_rw_unlock,
 4348         [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private_compat32,
 4349         [UMTX_OP_WAKE_PRIVATE]  = __umtx_op_wake_private,
 4350         [UMTX_OP_MUTEX_WAIT]    = __umtx_op_wait_umutex_compat32,
 4351         [UMTX_OP_MUTEX_WAKE]    = __umtx_op_wake_umutex,
 4352 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
 4353         [UMTX_OP_SEM_WAIT]      = __umtx_op_sem_wait_compat32,
 4354         [UMTX_OP_SEM_WAKE]      = __umtx_op_sem_wake,
 4355 #else
 4356         [UMTX_OP_SEM_WAIT]      = __umtx_op_unimpl,
 4357         [UMTX_OP_SEM_WAKE]      = __umtx_op_unimpl,
 4358 #endif
 4359         [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private32,
 4360         [UMTX_OP_MUTEX_WAKE2]   = __umtx_op_wake2_umutex,
 4361         [UMTX_OP_SEM2_WAIT]     = __umtx_op_sem2_wait_compat32,
 4362         [UMTX_OP_SEM2_WAKE]     = __umtx_op_sem2_wake,
 4363         [UMTX_OP_SHM]           = __umtx_op_shm,
 4364         [UMTX_OP_ROBUST_LISTS]  = __umtx_op_robust_lists_compat32,
 4365 };
 4366 
 4367 int
 4368 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
 4369 {
 4370 
 4371         if ((unsigned)uap->op < nitems(op_table_compat32)) {
 4372                 return (*op_table_compat32[uap->op])(td,
 4373                     (struct _umtx_op_args *)uap);
 4374         }
 4375         return (EINVAL);
 4376 }
 4377 #endif
 4378 
 4379 void
 4380 umtx_thread_init(struct thread *td)
 4381 {
 4382 
 4383         td->td_umtxq = umtxq_alloc();
 4384         td->td_umtxq->uq_thread = td;
 4385 }
 4386 
 4387 void
 4388 umtx_thread_fini(struct thread *td)
 4389 {
 4390 
 4391         umtxq_free(td->td_umtxq);
 4392 }
 4393 
 4394 /*
 4395  * It will be called when new thread is created, e.g fork().
 4396  */
 4397 void
 4398 umtx_thread_alloc(struct thread *td)
 4399 {
 4400         struct umtx_q *uq;
 4401 
 4402         uq = td->td_umtxq;
 4403         uq->uq_inherited_pri = PRI_MAX;
 4404 
 4405         KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
 4406         KASSERT(uq->uq_thread == td, ("uq_thread != td"));
 4407         KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
 4408         KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
 4409 }
 4410 
 4411 /*
 4412  * exec() hook.
 4413  *
 4414  * Clear robust lists for all process' threads, not delaying the
 4415  * cleanup to thread_exit hook, since the relevant address space is
 4416  * destroyed right now.
 4417  */
 4418 static void
 4419 umtx_exec_hook(void *arg __unused, struct proc *p,
 4420     struct image_params *imgp __unused)
 4421 {
 4422         struct thread *td;
 4423 
 4424         KASSERT(p == curproc, ("need curproc"));
 4425         PROC_LOCK(p);
 4426         KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
 4427             (p->p_flag & P_STOPPED_SINGLE) != 0,
 4428             ("curproc must be single-threaded"));
 4429         FOREACH_THREAD_IN_PROC(p, td) {
 4430                 KASSERT(td == curthread ||
 4431                     ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
 4432                     ("running thread %p %p", p, td));
 4433                 PROC_UNLOCK(p);
 4434                 umtx_thread_cleanup(td);
 4435                 PROC_LOCK(p);
 4436                 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
 4437         }
 4438         PROC_UNLOCK(p);
 4439 }
 4440 
 4441 /*
 4442  * thread_exit() hook.
 4443  */
 4444 void
 4445 umtx_thread_exit(struct thread *td)
 4446 {
 4447 
 4448         umtx_thread_cleanup(td);
 4449 }
 4450 
 4451 static int
 4452 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res)
 4453 {
 4454         u_long res1;
 4455 #ifdef COMPAT_FREEBSD32
 4456         uint32_t res32;
 4457 #endif
 4458         int error;
 4459 
 4460 #ifdef COMPAT_FREEBSD32
 4461         if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 4462                 error = fueword32((void *)ptr, &res32);
 4463                 if (error == 0)
 4464                         res1 = res32;
 4465         } else
 4466 #endif
 4467         {
 4468                 error = fueword((void *)ptr, &res1);
 4469         }
 4470         if (error == 0)
 4471                 *res = res1;
 4472         else
 4473                 error = EFAULT;
 4474         return (error);
 4475 }
 4476 
 4477 static void
 4478 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list)
 4479 {
 4480 #ifdef COMPAT_FREEBSD32
 4481         struct umutex32 m32;
 4482 
 4483         if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
 4484                 memcpy(&m32, m, sizeof(m32));
 4485                 *rb_list = m32.m_rb_lnk;
 4486         } else
 4487 #endif
 4488                 *rb_list = m->m_rb_lnk;
 4489 }
 4490 
 4491 static int
 4492 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact)
 4493 {
 4494         struct umutex m;
 4495         int error;
 4496 
 4497         KASSERT(td->td_proc == curproc, ("need current vmspace"));
 4498         error = copyin((void *)rbp, &m, sizeof(m));
 4499         if (error != 0)
 4500                 return (error);
 4501         if (rb_list != NULL)
 4502                 umtx_read_rb_list(td, &m, rb_list);
 4503         if ((m.m_flags & UMUTEX_ROBUST) == 0)
 4504                 return (EINVAL);
 4505         if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
 4506                 /* inact is cleared after unlock, allow the inconsistency */
 4507                 return (inact ? 0 : EINVAL);
 4508         return (do_unlock_umutex(td, (struct umutex *)rbp, true));
 4509 }
 4510 
 4511 static void
 4512 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact,
 4513     const char *name)
 4514 {
 4515         int error, i;
 4516         uintptr_t rbp;
 4517         bool inact;
 4518 
 4519         if (rb_list == 0)
 4520                 return;
 4521         error = umtx_read_uptr(td, rb_list, &rbp);
 4522         for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) {
 4523                 if (rbp == *rb_inact) {
 4524                         inact = true;
 4525                         *rb_inact = 0;
 4526                 } else
 4527                         inact = false;
 4528                 error = umtx_handle_rb(td, rbp, &rbp, inact);
 4529         }
 4530         if (i == umtx_max_rb && umtx_verbose_rb) {
 4531                 uprintf("comm %s pid %d: reached umtx %smax rb %d\n",
 4532                     td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
 4533         }
 4534         if (error != 0 && umtx_verbose_rb) {
 4535                 uprintf("comm %s pid %d: handling %srb error %d\n",
 4536                     td->td_proc->p_comm, td->td_proc->p_pid, name, error);
 4537         }
 4538 }
 4539 
 4540 /*
 4541  * Clean up umtx data.
 4542  */
 4543 static void
 4544 umtx_thread_cleanup(struct thread *td)
 4545 {
 4546         struct umtx_q *uq;
 4547         struct umtx_pi *pi;
 4548         uintptr_t rb_inact;
 4549 
 4550         /*
 4551          * Disown pi mutexes.
 4552          */
 4553         uq = td->td_umtxq;
 4554         if (uq != NULL) {
 4555                 mtx_lock(&umtx_lock);
 4556                 uq->uq_inherited_pri = PRI_MAX;
 4557                 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
 4558                         pi->pi_owner = NULL;
 4559                         TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
 4560                 }
 4561                 mtx_unlock(&umtx_lock);
 4562                 thread_lock(td);
 4563                 sched_lend_user_prio(td, PRI_MAX);
 4564                 thread_unlock(td);
 4565         }
 4566 
 4567         /*
 4568          * Handle terminated robust mutexes.  Must be done after
 4569          * robust pi disown, otherwise unlock could see unowned
 4570          * entries.
 4571          */
 4572         rb_inact = td->td_rb_inact;
 4573         if (rb_inact != 0)
 4574                 (void)umtx_read_uptr(td, rb_inact, &rb_inact);
 4575         umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "");
 4576         umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ");
 4577         if (rb_inact != 0)
 4578                 (void)umtx_handle_rb(td, rb_inact, NULL, true);
 4579 }

Cache object: 03d650ce20d3b78c0e2bbfa5288dcac5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.