The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/compat/linux/linux_futex.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */
    2 
    3 /*-
    4  * Copyright (c) 2009-2016 Dmitry Chagin
    5  * Copyright (c) 2005 Emmanuel Dreyfus
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by Emmanuel Dreyfus
   19  * 4. The name of the author may not be used to endorse or promote
   20  *    products derived from this software without specific prior written
   21  *    permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS''
   24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
   25  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   26  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
   27  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   33  * POSSIBILITY OF SUCH DAMAGE.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD: releng/11.2/sys/compat/linux/linux_futex.c 317936 2017-05-08 10:51:30Z dchagin $");
   38 #if 0
   39 __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $");
   40 #endif
   41 
   42 #include "opt_compat.h"
   43 
   44 #include <sys/param.h>
   45 #include <sys/systm.h>
   46 #include <sys/imgact.h>
   47 #include <sys/kernel.h>
   48 #include <sys/ktr.h>
   49 #include <sys/lock.h>
   50 #include <sys/malloc.h>
   51 #include <sys/mutex.h>
   52 #include <sys/priv.h>
   53 #include <sys/proc.h>
   54 #include <sys/queue.h>
   55 #include <sys/sched.h>
   56 #include <sys/sdt.h>
   57 #include <sys/umtx.h>
   58 
   59 #include <vm/vm_extern.h>
   60 
   61 #ifdef COMPAT_LINUX32
   62 #include <machine/../linux32/linux.h>
   63 #include <machine/../linux32/linux32_proto.h>
   64 #else
   65 #include <machine/../linux/linux.h>
   66 #include <machine/../linux/linux_proto.h>
   67 #endif
   68 #include <compat/linux/linux_dtrace.h>
   69 #include <compat/linux/linux_emul.h>
   70 #include <compat/linux/linux_futex.h>
   71 #include <compat/linux/linux_timer.h>
   72 #include <compat/linux/linux_util.h>
   73 
   74 /* DTrace init */
   75 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
   76 
   77 /**
   78  * Futex part for the special DTrace module "locks".
   79  */
   80 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *");
   81 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *");
   82 
   83 /**
   84  * Per futex probes.
   85  */
   86 LIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *");
   87 LIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *");
   88 
   89 /**
   90  * DTrace probes in this module.
   91  */
   92 LIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *",
   93     "struct waiting_proc *");
   94 LIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t",
   95     "int");
   96 LIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t",
   97     "int");
   98 LIN_SDT_PROBE_DEFINE0(futex, futex_put, return);
   99 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **",
  100     "uint32_t");
  101 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int");
  102 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t",
  103     "int");
  104 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *");
  105 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int");
  106 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int");
  107 LIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *",
  108     "struct waiting_proc **", "struct futex **");
  109 LIN_SDT_PROBE_DEFINE0(futex, futex_get, error);
  110 LIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int");
  111 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *",
  112     "struct waiting_proc **", "struct timespec *");
  113 LIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *",
  114     "struct waiting_proc *", "uint32_t *", "uint32_t");
  115 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *",
  116     "struct waiting_proc *");
  117 LIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int");
  118 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int",
  119     "uint32_t");
  120 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t",
  121     "struct waiting_proc *", "uint32_t");
  122 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *");
  123 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int");
  124 LIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int",
  125     "struct futex *", "int");
  126 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *");
  127 LIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *",
  128     "struct waiting_proc *", "uint32_t");
  129 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int");
  130 LIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *",
  131     "struct waiting_proc **", "struct timespec *", "uint32_t");
  132 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int");
  133 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int");
  134 LIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *",
  135     "int", "uint32_t");
  136 LIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int",
  137     "int");
  138 LIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check);
  139 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int");
  140 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int");
  141 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int");
  142 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *",
  143     "struct linux_sys_futex_args *");
  144 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch);
  145 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int");
  146 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use);
  147 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *",
  148     "uint32_t", "uint32_t");
  149 LIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq,
  150     "uint32_t *", "uint32_t", "int", "uint32_t");
  151 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *",
  152     "uint32_t", "uint32_t");
  153 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *",
  154     "uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *");
  155 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq,
  156     "uint32_t", "int");
  157 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *",
  158     "int", "uint32_t", "uint32_t *", "uint32_t");
  159 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault);
  160 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi);
  161 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi);
  162 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi);
  163 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue);
  164 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi);
  165 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi);
  166 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int");
  167 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int");
  168 LIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *",
  169     "struct linux_set_robust_list_args *");
  170 LIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error);
  171 LIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int");
  172 LIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *",
  173     "struct linux_get_robust_list_args *");
  174 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int");
  175 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int");
  176 LIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry,
  177     "struct linux_emuldata *", "uint32_t *", "unsigned int");
  178 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int");
  179 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int");
  180 LIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry,
  181     "struct linux_robust_list **", "struct linux_robust_list **",
  182     "unsigned int *");
  183 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int");
  184 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int");
  185 LIN_SDT_PROBE_DEFINE2(futex, release_futexes, entry, "struct thread *",
  186     "struct linux_emuldata *");
  187 LIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int");
  188 LIN_SDT_PROBE_DEFINE0(futex, release_futexes, return);
  189 
  190 struct futex;
  191 
  192 struct waiting_proc {
  193         uint32_t        wp_flags;
  194         struct futex    *wp_futex;
  195         TAILQ_ENTRY(waiting_proc) wp_list;
  196 };
  197 
  198 struct futex {
  199         struct mtx      f_lck;
  200         uint32_t        *f_uaddr;       /* user-supplied value, for debug */
  201         struct umtx_key f_key;
  202         uint32_t        f_refcount;
  203         uint32_t        f_bitset;
  204         LIST_ENTRY(futex) f_list;
  205         TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc;
  206 };
  207 
  208 struct futex_list futex_list;
  209 
  210 #define FUTEX_LOCK(f)           mtx_lock(&(f)->f_lck)
  211 #define FUTEX_LOCKED(f)         mtx_owned(&(f)->f_lck)
  212 #define FUTEX_UNLOCK(f)         mtx_unlock(&(f)->f_lck)
  213 #define FUTEX_INIT(f)           do { \
  214                                     mtx_init(&(f)->f_lck, "ftlk", NULL, \
  215                                         MTX_DUPOK); \
  216                                     LIN_SDT_PROBE1(futex, futex, create, \
  217                                         &(f)->f_lck); \
  218                                 } while (0)
  219 #define FUTEX_DESTROY(f)        do { \
  220                                     LIN_SDT_PROBE1(futex, futex, destroy, \
  221                                         &(f)->f_lck); \
  222                                     mtx_destroy(&(f)->f_lck); \
  223                                 } while (0)
  224 #define FUTEX_ASSERT_LOCKED(f)  mtx_assert(&(f)->f_lck, MA_OWNED)
  225 #define FUTEX_ASSERT_UNLOCKED(f) mtx_assert(&(f)->f_lck, MA_NOTOWNED)
  226 
  227 struct mtx futex_mtx;                   /* protects the futex list */
  228 #define FUTEXES_LOCK            do { \
  229                                     mtx_lock(&futex_mtx); \
  230                                     LIN_SDT_PROBE1(locks, futex_mtx, \
  231                                         locked, &futex_mtx); \
  232                                 } while (0)
  233 #define FUTEXES_UNLOCK          do { \
  234                                     LIN_SDT_PROBE1(locks, futex_mtx, \
  235                                         unlock, &futex_mtx); \
  236                                     mtx_unlock(&futex_mtx); \
  237                                 } while (0)
  238 
  239 /* flags for futex_get() */
  240 #define FUTEX_CREATE_WP         0x1     /* create waiting_proc */
  241 #define FUTEX_DONTCREATE        0x2     /* don't create futex if not exists */
  242 #define FUTEX_DONTEXISTS        0x4     /* return EINVAL if futex exists */
  243 #define FUTEX_SHARED            0x8     /* shared futex */
  244 #define FUTEX_DONTLOCK          0x10    /* don't lock futex */
  245 
  246 /* wp_flags */
  247 #define FUTEX_WP_REQUEUED       0x1     /* wp requeued - wp moved from wp_list
  248                                          * of futex where thread sleep to wp_list
  249                                          * of another futex.
  250                                          */
  251 #define FUTEX_WP_REMOVED        0x2     /* wp is woken up and removed from futex
  252                                          * wp_list to prevent double wakeup.
  253                                          */
  254 
  255 static void futex_put(struct futex *, struct waiting_proc *);
  256 static int futex_get0(uint32_t *, struct futex **f, uint32_t);
  257 static int futex_get(uint32_t *, struct waiting_proc **, struct futex **,
  258     uint32_t);
  259 static int futex_sleep(struct futex *, struct waiting_proc *, struct timespec *);
  260 static int futex_wake(struct futex *, int, uint32_t);
  261 static int futex_requeue(struct futex *, int, struct futex *, int);
  262 static int futex_copyin_timeout(int, struct l_timespec *, int,
  263     struct timespec *);
  264 static int futex_wait(struct futex *, struct waiting_proc *, struct timespec *,
  265     uint32_t);
  266 static void futex_lock(struct futex *);
  267 static void futex_unlock(struct futex *);
  268 static int futex_atomic_op(struct thread *, int, uint32_t *);
  269 static int handle_futex_death(struct linux_emuldata *, uint32_t *,
  270     unsigned int);
  271 static int fetch_robust_entry(struct linux_robust_list **,
  272     struct linux_robust_list **, unsigned int *);
  273 
  274 /* support.s */
  275 int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval);
  276 int futex_addl(int oparg, uint32_t *uaddr, int *oldval);
  277 int futex_orl(int oparg, uint32_t *uaddr, int *oldval);
  278 int futex_andl(int oparg, uint32_t *uaddr, int *oldval);
  279 int futex_xorl(int oparg, uint32_t *uaddr, int *oldval);
  280 
  281 
  282 static int
  283 futex_copyin_timeout(int op, struct l_timespec *luts, int clockrt,
  284     struct timespec *ts)
  285 {
  286         struct l_timespec lts;
  287         struct timespec kts;
  288         int error;
  289 
  290         error = copyin(luts, &lts, sizeof(lts));
  291         if (error)
  292                 return (error);
  293 
  294         error = linux_to_native_timespec(ts, &lts);
  295         if (error)
  296                 return (error);
  297         if (clockrt) {
  298                 nanotime(&kts);
  299                 timespecsub(ts, &kts);
  300         } else if (op == LINUX_FUTEX_WAIT_BITSET) {
  301                 nanouptime(&kts);
  302                 timespecsub(ts, &kts);
  303         }
  304         return (error);
  305 }
  306 
  307 static void
  308 futex_put(struct futex *f, struct waiting_proc *wp)
  309 {
  310         LIN_SDT_PROBE2(futex, futex_put, entry, f, wp);
  311 
  312         if (wp != NULL) {
  313                 if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0)
  314                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  315                 free(wp, M_FUTEX_WP);
  316         }
  317 
  318         FUTEXES_LOCK;
  319         if (--f->f_refcount == 0) {
  320                 LIST_REMOVE(f, f_list);
  321                 FUTEXES_UNLOCK;
  322                 if (FUTEX_LOCKED(f))
  323                         futex_unlock(f);
  324 
  325                 LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr,
  326                     f->f_refcount, f->f_key.shared);
  327                 LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d "
  328                     "shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared);
  329                 umtx_key_release(&f->f_key);
  330                 FUTEX_DESTROY(f);
  331                 free(f, M_FUTEX);
  332 
  333                 LIN_SDT_PROBE0(futex, futex_put, return);
  334                 return;
  335         }
  336 
  337         LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount,
  338             f->f_key.shared);
  339         LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d",
  340             f->f_uaddr, f->f_refcount, f->f_key.shared);
  341         FUTEXES_UNLOCK;
  342         if (FUTEX_LOCKED(f))
  343                 futex_unlock(f);
  344 
  345         LIN_SDT_PROBE0(futex, futex_put, return);
  346 }
  347 
  348 static int
  349 futex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags)
  350 {
  351         struct futex *f, *tmpf;
  352         struct umtx_key key;
  353         int error;
  354 
  355         LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags);
  356 
  357         *newf = tmpf = NULL;
  358 
  359         error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ?
  360             AUTO_SHARE : THREAD_SHARE, &key);
  361         if (error) {
  362                 LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error);
  363                 LIN_SDT_PROBE1(futex, futex_get0, return, error);
  364                 return (error);
  365         }
  366 retry:
  367         FUTEXES_LOCK;
  368         LIST_FOREACH(f, &futex_list, f_list) {
  369                 if (umtx_key_match(&f->f_key, &key)) {
  370                         if (tmpf != NULL) {
  371                                 if (FUTEX_LOCKED(tmpf))
  372                                         futex_unlock(tmpf);
  373                                 FUTEX_DESTROY(tmpf);
  374                                 free(tmpf, M_FUTEX);
  375                         }
  376                         if (flags & FUTEX_DONTEXISTS) {
  377                                 FUTEXES_UNLOCK;
  378                                 umtx_key_release(&key);
  379 
  380                                 LIN_SDT_PROBE1(futex, futex_get0, return,
  381                                     EINVAL);
  382                                 return (EINVAL);
  383                         }
  384 
  385                         /*
  386                          * Increment refcount of the found futex to
  387                          * prevent it from deallocation before FUTEX_LOCK()
  388                          */
  389                         ++f->f_refcount;
  390                         FUTEXES_UNLOCK;
  391                         umtx_key_release(&key);
  392 
  393                         if ((flags & FUTEX_DONTLOCK) == 0)
  394                                 futex_lock(f);
  395                         *newf = f;
  396                         LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr,
  397                             f->f_refcount, f->f_key.shared);
  398                         LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d",
  399                             uaddr, f->f_refcount, f->f_key.shared);
  400 
  401                         LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  402                         return (0);
  403                 }
  404         }
  405 
  406         if (flags & FUTEX_DONTCREATE) {
  407                 FUTEXES_UNLOCK;
  408                 umtx_key_release(&key);
  409                 LIN_SDT_PROBE1(futex, futex_get0, null, uaddr);
  410                 LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr);
  411 
  412                 LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  413                 return (0);
  414         }
  415 
  416         if (tmpf == NULL) {
  417                 FUTEXES_UNLOCK;
  418                 tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO);
  419                 tmpf->f_uaddr = uaddr;
  420                 tmpf->f_key = key;
  421                 tmpf->f_refcount = 1;
  422                 tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY;
  423                 FUTEX_INIT(tmpf);
  424                 TAILQ_INIT(&tmpf->f_waiting_proc);
  425 
  426                 /*
  427                  * Lock the new futex before an insert into the futex_list
  428                  * to prevent futex usage by other.
  429                  */
  430                 if ((flags & FUTEX_DONTLOCK) == 0)
  431                         futex_lock(tmpf);
  432                 goto retry;
  433         }
  434 
  435         LIST_INSERT_HEAD(&futex_list, tmpf, f_list);
  436         FUTEXES_UNLOCK;
  437 
  438         LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount,
  439             tmpf->f_key.shared);
  440         LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new",
  441             uaddr, tmpf->f_refcount, tmpf->f_key.shared);
  442         *newf = tmpf;
  443 
  444         LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  445         return (0);
  446 }
  447 
  448 static int
  449 futex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f,
  450     uint32_t flags)
  451 {
  452         int error;
  453 
  454         LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f);
  455 
  456         if (flags & FUTEX_CREATE_WP) {
  457                 *wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK);
  458                 (*wp)->wp_flags = 0;
  459         }
  460         error = futex_get0(uaddr, f, flags);
  461         if (error) {
  462                 LIN_SDT_PROBE0(futex, futex_get, error);
  463 
  464                 if (flags & FUTEX_CREATE_WP)
  465                         free(*wp, M_FUTEX_WP);
  466 
  467                 LIN_SDT_PROBE1(futex, futex_get, return, error);
  468                 return (error);
  469         }
  470         if (flags & FUTEX_CREATE_WP) {
  471                 TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list);
  472                 (*wp)->wp_futex = *f;
  473         }
  474 
  475         LIN_SDT_PROBE1(futex, futex_get, return, error);
  476         return (error);
  477 }
  478 
  479 static inline void
  480 futex_lock(struct futex *f)
  481 {
  482 
  483         LINUX_CTR3(sys_futex, "futex_lock uaddr %p ref %d shared %d",
  484             f->f_uaddr, f->f_refcount, f->f_key.shared);
  485         FUTEX_ASSERT_UNLOCKED(f);
  486         FUTEX_LOCK(f);
  487 }
  488 
  489 static inline void
  490 futex_unlock(struct futex *f)
  491 {
  492 
  493         LINUX_CTR3(sys_futex, "futex_unlock uaddr %p ref %d shared %d",
  494             f->f_uaddr, f->f_refcount, f->f_key.shared);
  495         FUTEX_ASSERT_LOCKED(f);
  496         FUTEX_UNLOCK(f);
  497 }
  498 
  499 static int
  500 futex_sleep(struct futex *f, struct waiting_proc *wp, struct timespec *ts)
  501 {
  502         struct timespec uts;
  503         sbintime_t sbt, prec, tmp;
  504         time_t over;
  505         int error;
  506 
  507         FUTEX_ASSERT_LOCKED(f);
  508         if (ts != NULL) {
  509                 uts = *ts;
  510                 if (uts.tv_sec > INT32_MAX / 2) {
  511                         over = uts.tv_sec - INT32_MAX / 2;
  512                         uts.tv_sec -= over;
  513                 }
  514                 tmp = tstosbt(uts);
  515                 if (TIMESEL(&sbt, tmp))
  516                         sbt += tc_tick_sbt;
  517                 sbt += tmp;
  518                 prec = tmp;
  519                 prec >>= tc_precexp;
  520         } else {
  521                 sbt = 0;
  522                 prec = 0;
  523         }
  524         LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, sbt);
  525         LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %ld ref %d",
  526             f->f_uaddr, wp, sbt, f->f_refcount);
  527 
  528         error = msleep_sbt(wp, &f->f_lck, PCATCH, "futex", sbt, prec, C_ABSOLUTE);
  529         if (wp->wp_flags & FUTEX_WP_REQUEUED) {
  530                 KASSERT(f != wp->wp_futex, ("futex != wp_futex"));
  531 
  532                 if (error) {
  533                         LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error,
  534                             f->f_uaddr, wp, wp->wp_futex->f_uaddr,
  535                             wp->wp_futex->f_refcount);
  536                 }
  537 
  538                 LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp"
  539                     " %p requeued uaddr %p ref %d",
  540                     error, f->f_uaddr, wp, wp->wp_futex->f_uaddr,
  541                     wp->wp_futex->f_refcount);
  542                 futex_put(f, NULL);
  543                 f = wp->wp_futex;
  544                 futex_lock(f);
  545         } else {
  546                 if (error) {
  547                         LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error,
  548                             f->f_uaddr, wp);
  549                 }
  550                 LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p",
  551                     error, f->f_uaddr, wp);
  552         }
  553 
  554         futex_put(f, wp);
  555 
  556         LIN_SDT_PROBE1(futex, futex_sleep, return, error);
  557         return (error);
  558 }
  559 
  560 static int
  561 futex_wake(struct futex *f, int n, uint32_t bitset)
  562 {
  563         struct waiting_proc *wp, *wpt;
  564         int count = 0;
  565 
  566         LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset);
  567 
  568         if (bitset == 0) {
  569                 LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL);
  570                 return (EINVAL);
  571         }
  572 
  573         FUTEX_ASSERT_LOCKED(f);
  574         TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
  575                 LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp,
  576                     f->f_refcount);
  577                 LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d",
  578                     f->f_uaddr, wp, f->f_refcount);
  579                 /*
  580                  * Unless we find a matching bit in
  581                  * the bitset, continue searching.
  582                  */
  583                 if (!(wp->wp_futex->f_bitset & bitset))
  584                         continue;
  585 
  586                 wp->wp_flags |= FUTEX_WP_REMOVED;
  587                 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  588                 LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp);
  589                 wakeup_one(wp);
  590                 if (++count == n)
  591                         break;
  592         }
  593 
  594         LIN_SDT_PROBE1(futex, futex_wake, return, count);
  595         return (count);
  596 }
  597 
  598 static int
  599 futex_requeue(struct futex *f, int n, struct futex *f2, int n2)
  600 {
  601         struct waiting_proc *wp, *wpt;
  602         int count = 0;
  603 
  604         LIN_SDT_PROBE4(futex, futex_requeue, entry, f, n, f2, n2);
  605 
  606         FUTEX_ASSERT_LOCKED(f);
  607         FUTEX_ASSERT_LOCKED(f2);
  608 
  609         TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
  610                 if (++count <= n) {
  611                         LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p",
  612                             f->f_uaddr, wp);
  613                         wp->wp_flags |= FUTEX_WP_REMOVED;
  614                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  615                         LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp);
  616                         wakeup_one(wp);
  617                 } else {
  618                         LIN_SDT_PROBE3(futex, futex_requeue, requeue,
  619                             f->f_uaddr, wp, f2->f_uaddr);
  620                         LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p",
  621                             f->f_uaddr, wp, f2->f_uaddr);
  622                         wp->wp_flags |= FUTEX_WP_REQUEUED;
  623                         /* Move wp to wp_list of f2 futex */
  624                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  625                         TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list);
  626 
  627                         /*
  628                          * Thread which sleeps on wp after waking should
  629                          * acquire f2 lock, so increment refcount of f2 to
  630                          * prevent it from premature deallocation.
  631                          */
  632                         wp->wp_futex = f2;
  633                         FUTEXES_LOCK;
  634                         ++f2->f_refcount;
  635                         FUTEXES_UNLOCK;
  636                         if (count - n >= n2)
  637                                 break;
  638                 }
  639         }
  640 
  641         LIN_SDT_PROBE1(futex, futex_requeue, return, count);
  642         return (count);
  643 }
  644 
  645 static int
  646 futex_wait(struct futex *f, struct waiting_proc *wp, struct timespec *ts,
  647     uint32_t bitset)
  648 {
  649         int error;
  650 
  651         LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, ts, bitset);
  652 
  653         if (bitset == 0) {
  654                 LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL);
  655                 return (EINVAL);
  656         }
  657 
  658         f->f_bitset = bitset;
  659         error = futex_sleep(f, wp, ts);
  660         if (error)
  661                 LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error);
  662         if (error == EWOULDBLOCK)
  663                 error = ETIMEDOUT;
  664 
  665         LIN_SDT_PROBE1(futex, futex_wait, return, error);
  666         return (error);
  667 }
  668 
  669 static int
  670 futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr)
  671 {
  672         int op = (encoded_op >> 28) & 7;
  673         int cmp = (encoded_op >> 24) & 15;
  674         int oparg = (encoded_op << 8) >> 20;
  675         int cmparg = (encoded_op << 20) >> 20;
  676         int oldval = 0, ret;
  677 
  678         LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr);
  679 
  680         if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
  681                 oparg = 1 << oparg;
  682 
  683         LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg,
  684             cmparg);
  685 
  686         /* XXX: Linux verifies access here and returns EFAULT */
  687         LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check);
  688 
  689         switch (op) {
  690         case FUTEX_OP_SET:
  691                 ret = futex_xchgl(oparg, uaddr, &oldval);
  692                 break;
  693         case FUTEX_OP_ADD:
  694                 ret = futex_addl(oparg, uaddr, &oldval);
  695                 break;
  696         case FUTEX_OP_OR:
  697                 ret = futex_orl(oparg, uaddr, &oldval);
  698                 break;
  699         case FUTEX_OP_ANDN:
  700                 ret = futex_andl(~oparg, uaddr, &oldval);
  701                 break;
  702         case FUTEX_OP_XOR:
  703                 ret = futex_xorl(oparg, uaddr, &oldval);
  704                 break;
  705         default:
  706                 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op);
  707                 ret = -ENOSYS;
  708                 break;
  709         }
  710 
  711         if (ret) {
  712                 LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
  713                 return (ret);
  714         }
  715 
  716         switch (cmp) {
  717         case FUTEX_OP_CMP_EQ:
  718                 ret = (oldval == cmparg);
  719                 break;
  720         case FUTEX_OP_CMP_NE:
  721                 ret = (oldval != cmparg);
  722                 break;
  723         case FUTEX_OP_CMP_LT:
  724                 ret = (oldval < cmparg);
  725                 break;
  726         case FUTEX_OP_CMP_GE:
  727                 ret = (oldval >= cmparg);
  728                 break;
  729         case FUTEX_OP_CMP_LE:
  730                 ret = (oldval <= cmparg);
  731                 break;
  732         case FUTEX_OP_CMP_GT:
  733                 ret = (oldval > cmparg);
  734                 break;
  735         default:
  736                 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp);
  737                 ret = -ENOSYS;
  738         }
  739 
  740         LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
  741         return (ret);
  742 }
  743 
  744 int
  745 linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
  746 {
  747         int clockrt, nrwake, op_ret, ret;
  748         struct linux_pemuldata *pem;
  749         struct waiting_proc *wp;
  750         struct futex *f, *f2;
  751         struct timespec uts, *ts;
  752         int error, save;
  753         uint32_t flags, val;
  754 
  755         LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args);
  756 
  757         if (args->op & LINUX_FUTEX_PRIVATE_FLAG) {
  758                 flags = 0;
  759                 args->op &= ~LINUX_FUTEX_PRIVATE_FLAG;
  760         } else
  761                 flags = FUTEX_SHARED;
  762 
  763         /*
  764          * Currently support for switching between CLOCK_MONOTONIC and
  765          * CLOCK_REALTIME is not present. However Linux forbids the use of
  766          * FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and
  767          * FUTEX_WAIT_REQUEUE_PI.
  768          */
  769         clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME;
  770         args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME;
  771         if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET &&
  772                 args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) {
  773                 LIN_SDT_PROBE0(futex, linux_sys_futex,
  774                     unimplemented_clockswitch);
  775                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
  776                 return (ENOSYS);
  777         }
  778 
  779         error = 0;
  780         f = f2 = NULL;
  781 
  782         switch (args->op) {
  783         case LINUX_FUTEX_WAIT:
  784                 args->val3 = FUTEX_BITSET_MATCH_ANY;
  785                 /* FALLTHROUGH */
  786 
  787         case LINUX_FUTEX_WAIT_BITSET:
  788                 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr,
  789                     args->val, args->val3);
  790                 LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x",
  791                     args->uaddr, args->val, args->val3);
  792 
  793                 if (args->timeout != NULL) {
  794                         error = futex_copyin_timeout(args->op, args->timeout,
  795                             clockrt, &uts);
  796                         if (error) {
  797                                 LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  798                                     error);
  799                                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  800                                 return (error);
  801                         }
  802                         ts = &uts;
  803                 } else
  804                         ts = NULL;
  805 
  806 retry0:
  807                 error = futex_get(args->uaddr, &wp, &f,
  808                     flags | FUTEX_CREATE_WP);
  809                 if (error) {
  810                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  811                         return (error);
  812                 }
  813 
  814                 error = copyin_nofault(args->uaddr, &val, sizeof(val));
  815                 if (error) {
  816                         futex_put(f, wp);
  817                         error = copyin(args->uaddr, &val, sizeof(val));
  818                         if (error == 0)
  819                                 goto retry0;
  820                         LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  821                             error);
  822                         LINUX_CTR1(sys_futex, "WAIT copyin failed %d",
  823                             error);
  824                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  825                         return (error);
  826                 }
  827                 if (val != args->val) {
  828                         LIN_SDT_PROBE4(futex, linux_sys_futex,
  829                             debug_wait_value_neq, args->uaddr, args->val, val,
  830                             args->val3);
  831                         LINUX_CTR3(sys_futex,
  832                             "WAIT uaddr %p val 0x%x != uval 0x%x",
  833                             args->uaddr, args->val, val);
  834                         futex_put(f, wp);
  835 
  836                         LIN_SDT_PROBE1(futex, linux_sys_futex, return,
  837                             EWOULDBLOCK);
  838                         return (EWOULDBLOCK);
  839                 }
  840 
  841                 error = futex_wait(f, wp, ts, args->val3);
  842                 break;
  843 
  844         case LINUX_FUTEX_WAKE:
  845                 args->val3 = FUTEX_BITSET_MATCH_ANY;
  846                 /* FALLTHROUGH */
  847 
  848         case LINUX_FUTEX_WAKE_BITSET:
  849                 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr,
  850                     args->val, args->val3);
  851                 LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x",
  852                     args->uaddr, args->val, args->val3);
  853 
  854                 error = futex_get(args->uaddr, NULL, &f,
  855                     flags | FUTEX_DONTCREATE);
  856                 if (error) {
  857                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  858                         return (error);
  859                 }
  860 
  861                 if (f == NULL) {
  862                         td->td_retval[0] = 0;
  863 
  864                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  865                         return (error);
  866                 }
  867                 td->td_retval[0] = futex_wake(f, args->val, args->val3);
  868                 futex_put(f, NULL);
  869                 break;
  870 
  871         case LINUX_FUTEX_CMP_REQUEUE:
  872                 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue,
  873                     args->uaddr, args->val, args->val3, args->uaddr2,
  874                     args->timeout);
  875                 LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p "
  876                     "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x",
  877                     args->uaddr, args->val, args->val3, args->uaddr2,
  878                     args->timeout);
  879 
  880                 /*
  881                  * Linux allows this, we would not, it is an incorrect
  882                  * usage of declared ABI, so return EINVAL.
  883                  */
  884                 if (args->uaddr == args->uaddr2) {
  885                         LIN_SDT_PROBE0(futex, linux_sys_futex,
  886                             invalid_cmp_requeue_use);
  887                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
  888                         return (EINVAL);
  889                 }
  890 
  891 retry1:
  892                 error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK);
  893                 if (error) {
  894                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  895                         return (error);
  896                 }
  897 
  898                 /*
  899                  * To avoid deadlocks return EINVAL if second futex
  900                  * exists at this time.
  901                  *
  902                  * Glibc fall back to FUTEX_WAKE in case of any error
  903                  * returned by FUTEX_CMP_REQUEUE.
  904                  */
  905                 error = futex_get(args->uaddr2, NULL, &f2,
  906                     flags | FUTEX_DONTEXISTS | FUTEX_DONTLOCK);
  907                 if (error) {
  908                         futex_put(f, NULL);
  909 
  910                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  911                         return (error);
  912                 }
  913                 futex_lock(f);
  914                 futex_lock(f2);
  915                 error = copyin_nofault(args->uaddr, &val, sizeof(val));
  916                 if (error) {
  917                         futex_put(f2, NULL);
  918                         futex_put(f, NULL);
  919                         error = copyin(args->uaddr, &val, sizeof(val));
  920                         if (error == 0)
  921                                 goto retry1;
  922                         LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  923                             error);
  924                         LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d",
  925                             error);
  926                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  927                         return (error);
  928                 }
  929                 if (val != args->val3) {
  930                         LIN_SDT_PROBE2(futex, linux_sys_futex,
  931                             debug_cmp_requeue_value_neq, args->val, val);
  932                         LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x",
  933                             args->val, val);
  934                         futex_put(f2, NULL);
  935                         futex_put(f, NULL);
  936 
  937                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN);
  938                         return (EAGAIN);
  939                 }
  940 
  941                 nrwake = (int)(unsigned long)args->timeout;
  942                 td->td_retval[0] = futex_requeue(f, args->val, f2, nrwake);
  943                 futex_put(f2, NULL);
  944                 futex_put(f, NULL);
  945                 break;
  946 
  947         case LINUX_FUTEX_WAKE_OP:
  948                 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op,
  949                     args->uaddr, args->op, args->val, args->uaddr2, args->val3);
  950                 LINUX_CTR5(sys_futex, "WAKE_OP "
  951                     "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x",
  952                     args->uaddr, args->val, args->uaddr2, args->val3,
  953                     args->timeout);
  954 
  955                 if (args->uaddr == args->uaddr2) {
  956                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
  957                         return (EINVAL);
  958                 }
  959 
  960 retry2:
  961                 error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK);
  962                 if (error) {
  963                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  964                         return (error);
  965                 }
  966 
  967                 error = futex_get(args->uaddr2, NULL, &f2, flags | FUTEX_DONTLOCK);
  968                 if (error) {
  969                         futex_put(f, NULL);
  970 
  971                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  972                         return (error);
  973                 }
  974                 futex_lock(f);
  975                 futex_lock(f2);
  976 
  977                 /*
  978                  * This function returns positive number as results and
  979                  * negative as errors
  980                  */
  981                 save = vm_fault_disable_pagefaults();
  982                 op_ret = futex_atomic_op(td, args->val3, args->uaddr2);
  983                 vm_fault_enable_pagefaults(save);
  984 
  985                 LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x",
  986                     args->uaddr, op_ret);
  987 
  988                 if (op_ret < 0) {
  989                         if (f2 != NULL)
  990                                 futex_put(f2, NULL);
  991                         futex_put(f, NULL);
  992                         error = copyin(args->uaddr2, &val, sizeof(val));
  993                         if (error == 0)
  994                                 goto retry2;
  995                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  996                         return (error);
  997                 }
  998 
  999                 ret = futex_wake(f, args->val, args->val3);
 1000 
 1001                 if (op_ret > 0) {
 1002                         op_ret = 0;
 1003                         nrwake = (int)(unsigned long)args->timeout;
 1004 
 1005                         if (f2 != NULL)
 1006                                 op_ret += futex_wake(f2, nrwake, args->val3);
 1007                         else
 1008                                 op_ret += futex_wake(f, nrwake, args->val3);
 1009                         ret += op_ret;
 1010 
 1011                 }
 1012                 if (f2 != NULL)
 1013                         futex_put(f2, NULL);
 1014                 futex_put(f, NULL);
 1015                 td->td_retval[0] = ret;
 1016                 break;
 1017 
 1018         case LINUX_FUTEX_LOCK_PI:
 1019                 /* not yet implemented */
 1020                 pem = pem_find(td->td_proc);
 1021                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1022                         linux_msg(td,
 1023                                   "linux_sys_futex: "
 1024                                   "unsupported futex_pi op\n");
 1025                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1026                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1027                             unimplemented_lock_pi);
 1028                 }
 1029                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1030                 return (ENOSYS);
 1031 
 1032         case LINUX_FUTEX_UNLOCK_PI:
 1033                 /* not yet implemented */
 1034                 pem = pem_find(td->td_proc);
 1035                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1036                         linux_msg(td,
 1037                                   "linux_sys_futex: "
 1038                                   "unsupported futex_pi op\n");
 1039                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1040                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1041                             unimplemented_unlock_pi);
 1042                 }
 1043                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1044                 return (ENOSYS);
 1045 
 1046         case LINUX_FUTEX_TRYLOCK_PI:
 1047                 /* not yet implemented */
 1048                 pem = pem_find(td->td_proc);
 1049                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1050                         linux_msg(td,
 1051                                   "linux_sys_futex: "
 1052                                   "unsupported futex_pi op\n");
 1053                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1054                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1055                             unimplemented_trylock_pi);
 1056                 }
 1057                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1058                 return (ENOSYS);
 1059 
 1060         case LINUX_FUTEX_REQUEUE:
 1061                 /*
 1062                  * Glibc does not use this operation since version 2.3.3,
 1063                  * as it is racy and replaced by FUTEX_CMP_REQUEUE operation.
 1064                  * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when
 1065                  * FUTEX_REQUEUE returned EINVAL.
 1066                  */
 1067                 pem = pem_find(td->td_proc);
 1068                 if ((pem->flags & LINUX_XDEPR_REQUEUEOP) == 0) {
 1069                         linux_msg(td,
 1070                                   "linux_sys_futex: "
 1071                                   "unsupported futex_requeue op\n");
 1072                         pem->flags |= LINUX_XDEPR_REQUEUEOP;
 1073                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1074                             deprecated_requeue);
 1075                 }
 1076 
 1077                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
 1078                 return (EINVAL);
 1079 
 1080         case LINUX_FUTEX_WAIT_REQUEUE_PI:
 1081                 /* not yet implemented */
 1082                 pem = pem_find(td->td_proc);
 1083                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1084                         linux_msg(td,
 1085                                   "linux_sys_futex: "
 1086                                   "unsupported futex_pi op\n");
 1087                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1088                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1089                             unimplemented_wait_requeue_pi);
 1090                 }
 1091                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1092                 return (ENOSYS);
 1093 
 1094         case LINUX_FUTEX_CMP_REQUEUE_PI:
 1095                 /* not yet implemented */
 1096                 pem = pem_find(td->td_proc);
 1097                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1098                         linux_msg(td,
 1099                                   "linux_sys_futex: "
 1100                                   "unsupported futex_pi op\n");
 1101                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1102                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1103                             unimplemented_cmp_requeue_pi);
 1104                 }
 1105                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1106                 return (ENOSYS);
 1107 
 1108         default:
 1109                 linux_msg(td,
 1110                           "linux_sys_futex: unknown op %d\n", args->op);
 1111                 LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation,
 1112                     args->op);
 1113                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1114                 return (ENOSYS);
 1115         }
 1116 
 1117         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
 1118         return (error);
 1119 }
 1120 
 1121 int
 1122 linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args)
 1123 {
 1124         struct linux_emuldata *em;
 1125 
 1126         LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args);
 1127 
 1128         if (args->len != sizeof(struct linux_robust_list_head)) {
 1129                 LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error);
 1130                 LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL);
 1131                 return (EINVAL);
 1132         }
 1133 
 1134         em = em_find(td);
 1135         em->robust_futexes = args->head;
 1136 
 1137         LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0);
 1138         return (0);
 1139 }
 1140 
 1141 int
 1142 linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args)
 1143 {
 1144         struct linux_emuldata *em;
 1145         struct linux_robust_list_head *head;
 1146         l_size_t len = sizeof(struct linux_robust_list_head);
 1147         struct thread *td2;
 1148         int error = 0;
 1149 
 1150         LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args);
 1151 
 1152         if (!args->pid) {
 1153                 em = em_find(td);
 1154                 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
 1155                 head = em->robust_futexes;
 1156         } else {
 1157                 td2 = tdfind(args->pid, -1);
 1158                 if (td2 == NULL) {
 1159                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1160                             ESRCH);
 1161                         return (ESRCH);
 1162                 }
 1163                 if (SV_PROC_ABI(td2->td_proc) != SV_ABI_LINUX) {
 1164                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1165                             EPERM);
 1166                         PROC_UNLOCK(td2->td_proc);
 1167                         return (EPERM);
 1168                 }
 1169 
 1170                 em = em_find(td2);
 1171                 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
 1172                 /* XXX: ptrace? */
 1173                 if (priv_check(td, PRIV_CRED_SETUID) ||
 1174                     priv_check(td, PRIV_CRED_SETEUID) ||
 1175                     p_candebug(td, td2->td_proc)) {
 1176                         PROC_UNLOCK(td2->td_proc);
 1177 
 1178                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1179                             EPERM);
 1180                         return (EPERM);
 1181                 }
 1182                 head = em->robust_futexes;
 1183 
 1184                 PROC_UNLOCK(td2->td_proc);
 1185         }
 1186 
 1187         error = copyout(&len, args->len, sizeof(l_size_t));
 1188         if (error) {
 1189                 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
 1190                     error);
 1191                 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT);
 1192                 return (EFAULT);
 1193         }
 1194 
 1195         error = copyout(&head, args->head, sizeof(head));
 1196         if (error) {
 1197                 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
 1198                     error);
 1199         }
 1200 
 1201         LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error);
 1202         return (error);
 1203 }
 1204 
 1205 static int
 1206 handle_futex_death(struct linux_emuldata *em, uint32_t *uaddr,
 1207     unsigned int pi)
 1208 {
 1209         uint32_t uval, nval, mval;
 1210         struct futex *f;
 1211         int error;
 1212 
 1213         LIN_SDT_PROBE3(futex, handle_futex_death, entry, em, uaddr, pi);
 1214 
 1215 retry:
 1216         error = copyin(uaddr, &uval, 4);
 1217         if (error) {
 1218                 LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error);
 1219                 LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT);
 1220                 return (EFAULT);
 1221         }
 1222         if ((uval & FUTEX_TID_MASK) == em->em_tid) {
 1223                 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
 1224                 nval = casuword32(uaddr, uval, mval);
 1225 
 1226                 if (nval == -1) {
 1227                         LIN_SDT_PROBE1(futex, handle_futex_death, return,
 1228                             EFAULT);
 1229                         return (EFAULT);
 1230                 }
 1231 
 1232                 if (nval != uval)
 1233                         goto retry;
 1234 
 1235                 if (!pi && (uval & FUTEX_WAITERS)) {
 1236                         error = futex_get(uaddr, NULL, &f,
 1237                             FUTEX_DONTCREATE | FUTEX_SHARED);
 1238                         if (error) {
 1239                                 LIN_SDT_PROBE1(futex, handle_futex_death,
 1240                                     return, error);
 1241                                 return (error);
 1242                         }
 1243                         if (f != NULL) {
 1244                                 futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY);
 1245                                 futex_put(f, NULL);
 1246                         }
 1247                 }
 1248         }
 1249 
 1250         LIN_SDT_PROBE1(futex, handle_futex_death, return, 0);
 1251         return (0);
 1252 }
 1253 
 1254 static int
 1255 fetch_robust_entry(struct linux_robust_list **entry,
 1256     struct linux_robust_list **head, unsigned int *pi)
 1257 {
 1258         l_ulong uentry;
 1259         int error;
 1260 
 1261         LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi);
 1262 
 1263         error = copyin((const void *)head, &uentry, sizeof(l_ulong));
 1264         if (error) {
 1265                 LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error);
 1266                 LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT);
 1267                 return (EFAULT);
 1268         }
 1269 
 1270         *entry = (void *)(uentry & ~1UL);
 1271         *pi = uentry & 1;
 1272 
 1273         LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0);
 1274         return (0);
 1275 }
 1276 
 1277 /* This walks the list of robust futexes releasing them. */
 1278 void
 1279 release_futexes(struct thread *td, struct linux_emuldata *em)
 1280 {
 1281         struct linux_robust_list_head *head = NULL;
 1282         struct linux_robust_list *entry, *next_entry, *pending;
 1283         unsigned int limit = 2048, pi, next_pi, pip;
 1284         l_long futex_offset;
 1285         int rc, error;
 1286 
 1287         LIN_SDT_PROBE2(futex, release_futexes, entry, td, em);
 1288 
 1289         head = em->robust_futexes;
 1290 
 1291         if (head == NULL) {
 1292                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1293                 return;
 1294         }
 1295 
 1296         if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) {
 1297                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1298                 return;
 1299         }
 1300 
 1301         error = copyin(&head->futex_offset, &futex_offset,
 1302             sizeof(futex_offset));
 1303         if (error) {
 1304                 LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error);
 1305                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1306                 return;
 1307         }
 1308 
 1309         if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) {
 1310                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1311                 return;
 1312         }
 1313 
 1314         while (entry != &head->list) {
 1315                 rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi);
 1316 
 1317                 if (entry != pending)
 1318                         if (handle_futex_death(em,
 1319                             (uint32_t *)((caddr_t)entry + futex_offset), pi)) {
 1320                                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1321                                 return;
 1322                         }
 1323                 if (rc) {
 1324                         LIN_SDT_PROBE0(futex, release_futexes, return);
 1325                         return;
 1326                 }
 1327 
 1328                 entry = next_entry;
 1329                 pi = next_pi;
 1330 
 1331                 if (!--limit)
 1332                         break;
 1333 
 1334                 sched_relinquish(curthread);
 1335         }
 1336 
 1337         if (pending)
 1338                 handle_futex_death(em, (uint32_t *)((caddr_t)pending + futex_offset), pip);
 1339 
 1340         LIN_SDT_PROBE0(futex, release_futexes, return);
 1341 }

Cache object: bea4a9531b6979dfbcdbf47a1d01d487


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.