The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/compat/linux/linux_futex.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */
    2 
    3 /*-
    4  * Copyright (c) 2009-2016 Dmitry Chagin
    5  * Copyright (c) 2005 Emmanuel Dreyfus
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by Emmanuel Dreyfus
   19  * 4. The name of the author may not be used to endorse or promote
   20  *    products derived from this software without specific prior written
   21  *    permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS''
   24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
   25  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   26  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
   27  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   33  * POSSIBILITY OF SUCH DAMAGE.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD$");
   38 #if 0
   39 __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $");
   40 #endif
   41 
   42 #include "opt_compat.h"
   43 
   44 #include <sys/param.h>
   45 #include <sys/systm.h>
   46 #include <sys/imgact.h>
   47 #include <sys/kernel.h>
   48 #include <sys/ktr.h>
   49 #include <sys/lock.h>
   50 #include <sys/malloc.h>
   51 #include <sys/mutex.h>
   52 #include <sys/priv.h>
   53 #include <sys/proc.h>
   54 #include <sys/queue.h>
   55 #include <sys/sched.h>
   56 #include <sys/sdt.h>
   57 #include <sys/umtx.h>
   58 
   59 #include <vm/vm_extern.h>
   60 
   61 #ifdef COMPAT_LINUX32
   62 #include <machine/../linux32/linux.h>
   63 #include <machine/../linux32/linux32_proto.h>
   64 #else
   65 #include <machine/../linux/linux.h>
   66 #include <machine/../linux/linux_proto.h>
   67 #endif
   68 #include <compat/linux/linux_dtrace.h>
   69 #include <compat/linux/linux_emul.h>
   70 #include <compat/linux/linux_futex.h>
   71 #include <compat/linux/linux_timer.h>
   72 #include <compat/linux/linux_util.h>
   73 
   74 /* DTrace init */
   75 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
   76 
   77 /**
   78  * Futex part for the special DTrace module "locks".
   79  */
   80 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *");
   81 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *");
   82 
   83 /**
   84  * Per futex probes.
   85  */
   86 LIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *");
   87 LIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *");
   88 
   89 /**
   90  * DTrace probes in this module.
   91  */
   92 LIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *",
   93     "struct waiting_proc *");
   94 LIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t",
   95     "int");
   96 LIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t",
   97     "int");
   98 LIN_SDT_PROBE_DEFINE0(futex, futex_put, return);
   99 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **",
  100     "uint32_t");
  101 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int");
  102 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t",
  103     "int");
  104 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *");
  105 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int");
  106 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int");
  107 LIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *",
  108     "struct waiting_proc **", "struct futex **");
  109 LIN_SDT_PROBE_DEFINE0(futex, futex_get, error);
  110 LIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int");
  111 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *",
  112     "struct waiting_proc **", "struct timespec *");
  113 LIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *",
  114     "struct waiting_proc *", "uint32_t *", "uint32_t");
  115 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *",
  116     "struct waiting_proc *");
  117 LIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int");
  118 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int",
  119     "uint32_t");
  120 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t",
  121     "struct waiting_proc *", "uint32_t");
  122 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *");
  123 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int");
  124 LIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int",
  125     "struct futex *", "int");
  126 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *");
  127 LIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *",
  128     "struct waiting_proc *", "uint32_t");
  129 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int");
  130 LIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *",
  131     "struct waiting_proc **", "struct timespec *", "uint32_t");
  132 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int");
  133 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int");
  134 LIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *",
  135     "int", "uint32_t");
  136 LIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int",
  137     "int");
  138 LIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check);
  139 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int");
  140 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int");
  141 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int");
  142 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *",
  143     "struct linux_sys_futex_args *");
  144 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch);
  145 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int");
  146 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use);
  147 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *",
  148     "uint32_t", "uint32_t");
  149 LIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq,
  150     "uint32_t *", "uint32_t", "int", "uint32_t");
  151 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *",
  152     "uint32_t", "uint32_t");
  153 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *",
  154     "uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *");
  155 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq,
  156     "uint32_t", "int");
  157 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *",
  158     "int", "uint32_t", "uint32_t *", "uint32_t");
  159 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault);
  160 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi);
  161 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi);
  162 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi);
  163 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue);
  164 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi);
  165 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi);
  166 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int");
  167 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int");
  168 LIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *",
  169     "struct linux_set_robust_list_args *");
  170 LIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error);
  171 LIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int");
  172 LIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *",
  173     "struct linux_get_robust_list_args *");
  174 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int");
  175 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int");
  176 LIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry,
  177     "struct linux_emuldata *", "uint32_t *", "unsigned int");
  178 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int");
  179 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int");
  180 LIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry,
  181     "struct linux_robust_list **", "struct linux_robust_list **",
  182     "unsigned int *");
  183 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int");
  184 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int");
  185 LIN_SDT_PROBE_DEFINE2(futex, release_futexes, entry, "struct thread *",
  186     "struct linux_emuldata *");
  187 LIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int");
  188 LIN_SDT_PROBE_DEFINE0(futex, release_futexes, return);
  189 
  190 struct futex;
  191 
  192 struct waiting_proc {
  193         uint32_t        wp_flags;
  194         struct futex    *wp_futex;
  195         TAILQ_ENTRY(waiting_proc) wp_list;
  196 };
  197 
  198 struct futex {
  199         struct mtx      f_lck;
  200         uint32_t        *f_uaddr;       /* user-supplied value, for debug */
  201         struct umtx_key f_key;
  202         uint32_t        f_refcount;
  203         uint32_t        f_bitset;
  204         LIST_ENTRY(futex) f_list;
  205         TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc;
  206 };
  207 
  208 struct futex_list futex_list;
  209 
  210 #define FUTEX_LOCK(f)           mtx_lock(&(f)->f_lck)
  211 #define FUTEX_LOCKED(f)         mtx_owned(&(f)->f_lck)
  212 #define FUTEX_UNLOCK(f)         mtx_unlock(&(f)->f_lck)
  213 #define FUTEX_INIT(f)           do { \
  214                                     mtx_init(&(f)->f_lck, "ftlk", NULL, \
  215                                         MTX_DUPOK); \
  216                                     LIN_SDT_PROBE1(futex, futex, create, \
  217                                         &(f)->f_lck); \
  218                                 } while (0)
  219 #define FUTEX_DESTROY(f)        do { \
  220                                     LIN_SDT_PROBE1(futex, futex, destroy, \
  221                                         &(f)->f_lck); \
  222                                     mtx_destroy(&(f)->f_lck); \
  223                                 } while (0)
  224 #define FUTEX_ASSERT_LOCKED(f)  mtx_assert(&(f)->f_lck, MA_OWNED)
  225 #define FUTEX_ASSERT_UNLOCKED(f) mtx_assert(&(f)->f_lck, MA_NOTOWNED)
  226 
  227 struct mtx futex_mtx;                   /* protects the futex list */
  228 #define FUTEXES_LOCK            do { \
  229                                     mtx_lock(&futex_mtx); \
  230                                     LIN_SDT_PROBE1(locks, futex_mtx, \
  231                                         locked, &futex_mtx); \
  232                                 } while (0)
  233 #define FUTEXES_UNLOCK          do { \
  234                                     LIN_SDT_PROBE1(locks, futex_mtx, \
  235                                         unlock, &futex_mtx); \
  236                                     mtx_unlock(&futex_mtx); \
  237                                 } while (0)
  238 
  239 /* flags for futex_get() */
  240 #define FUTEX_CREATE_WP         0x1     /* create waiting_proc */
  241 #define FUTEX_DONTCREATE        0x2     /* don't create futex if not exists */
  242 #define FUTEX_DONTEXISTS        0x4     /* return EINVAL if futex exists */
  243 #define FUTEX_SHARED            0x8     /* shared futex */
  244 #define FUTEX_DONTLOCK          0x10    /* don't lock futex */
  245 
  246 /* wp_flags */
  247 #define FUTEX_WP_REQUEUED       0x1     /* wp requeued - wp moved from wp_list
  248                                          * of futex where thread sleep to wp_list
  249                                          * of another futex.
  250                                          */
  251 #define FUTEX_WP_REMOVED        0x2     /* wp is woken up and removed from futex
  252                                          * wp_list to prevent double wakeup.
  253                                          */
  254 
  255 static void futex_put(struct futex *, struct waiting_proc *);
  256 static int futex_get0(uint32_t *, struct futex **f, uint32_t);
  257 static int futex_get(uint32_t *, struct waiting_proc **, struct futex **,
  258     uint32_t);
  259 static int futex_sleep(struct futex *, struct waiting_proc *, struct timespec *);
  260 static int futex_wake(struct futex *, int, uint32_t);
  261 static int futex_requeue(struct futex *, int, struct futex *, int);
  262 static int futex_copyin_timeout(int, struct l_timespec *, int,
  263     struct timespec *);
  264 static int futex_wait(struct futex *, struct waiting_proc *, struct timespec *,
  265     uint32_t);
  266 static void futex_lock(struct futex *);
  267 static void futex_unlock(struct futex *);
  268 static int futex_atomic_op(struct thread *, int, uint32_t *);
  269 static int handle_futex_death(struct linux_emuldata *, uint32_t *,
  270     unsigned int);
  271 static int fetch_robust_entry(struct linux_robust_list **,
  272     struct linux_robust_list **, unsigned int *);
  273 
  274 /* support.s */
  275 int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval);
  276 int futex_addl(int oparg, uint32_t *uaddr, int *oldval);
  277 int futex_orl(int oparg, uint32_t *uaddr, int *oldval);
  278 int futex_andl(int oparg, uint32_t *uaddr, int *oldval);
  279 int futex_xorl(int oparg, uint32_t *uaddr, int *oldval);
  280 
  281 
  282 static int
  283 futex_copyin_timeout(int op, struct l_timespec *luts, int clockrt,
  284     struct timespec *ts)
  285 {
  286         struct l_timespec lts;
  287         struct timespec kts;
  288         int error;
  289 
  290         error = copyin(luts, &lts, sizeof(lts));
  291         if (error)
  292                 return (error);
  293 
  294         error = linux_to_native_timespec(ts, &lts);
  295         if (error)
  296                 return (error);
  297         if (clockrt) {
  298                 nanotime(&kts);
  299                 timespecsub(ts, &kts);
  300         } else if (op == LINUX_FUTEX_WAIT_BITSET) {
  301                 nanouptime(&kts);
  302                 timespecsub(ts, &kts);
  303         }
  304         return (error);
  305 }
  306 
  307 static void
  308 futex_put(struct futex *f, struct waiting_proc *wp)
  309 {
  310         LIN_SDT_PROBE2(futex, futex_put, entry, f, wp);
  311 
  312         if (wp != NULL) {
  313                 if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0)
  314                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  315                 free(wp, M_FUTEX_WP);
  316         }
  317 
  318         FUTEXES_LOCK;
  319         if (--f->f_refcount == 0) {
  320                 LIST_REMOVE(f, f_list);
  321                 FUTEXES_UNLOCK;
  322                 if (FUTEX_LOCKED(f))
  323                         futex_unlock(f);
  324 
  325                 LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr,
  326                     f->f_refcount, f->f_key.shared);
  327                 LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d "
  328                     "shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared);
  329                 umtx_key_release(&f->f_key);
  330                 FUTEX_DESTROY(f);
  331                 free(f, M_FUTEX);
  332 
  333                 LIN_SDT_PROBE0(futex, futex_put, return);
  334                 return;
  335         }
  336 
  337         LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount,
  338             f->f_key.shared);
  339         LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d",
  340             f->f_uaddr, f->f_refcount, f->f_key.shared);
  341         if (FUTEX_LOCKED(f))
  342                 futex_unlock(f);
  343         FUTEXES_UNLOCK;
  344 
  345         LIN_SDT_PROBE0(futex, futex_put, return);
  346 }
  347 
  348 static int
  349 futex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags)
  350 {
  351         struct futex *f, *tmpf;
  352         struct umtx_key key;
  353         int error;
  354 
  355         LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags);
  356 
  357         *newf = tmpf = NULL;
  358 
  359         error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ?
  360             AUTO_SHARE : THREAD_SHARE, &key);
  361         if (error) {
  362                 LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error);
  363                 LIN_SDT_PROBE1(futex, futex_get0, return, error);
  364                 return (error);
  365         }
  366 retry:
  367         FUTEXES_LOCK;
  368         LIST_FOREACH(f, &futex_list, f_list) {
  369                 if (umtx_key_match(&f->f_key, &key)) {
  370                         if (tmpf != NULL) {
  371                                 if (FUTEX_LOCKED(tmpf))
  372                                         futex_unlock(tmpf);
  373                                 FUTEX_DESTROY(tmpf);
  374                                 free(tmpf, M_FUTEX);
  375                         }
  376                         if (flags & FUTEX_DONTEXISTS) {
  377                                 FUTEXES_UNLOCK;
  378                                 umtx_key_release(&key);
  379 
  380                                 LIN_SDT_PROBE1(futex, futex_get0, return,
  381                                     EINVAL);
  382                                 return (EINVAL);
  383                         }
  384 
  385                         /*
  386                          * Increment refcount of the found futex to
  387                          * prevent it from deallocation before FUTEX_LOCK()
  388                          */
  389                         ++f->f_refcount;
  390                         FUTEXES_UNLOCK;
  391                         umtx_key_release(&key);
  392 
  393                         if ((flags & FUTEX_DONTLOCK) == 0)
  394                                 futex_lock(f);
  395                         *newf = f;
  396                         LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr,
  397                             f->f_refcount, f->f_key.shared);
  398                         LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d",
  399                             uaddr, f->f_refcount, f->f_key.shared);
  400 
  401                         LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  402                         return (0);
  403                 }
  404         }
  405 
  406         if (flags & FUTEX_DONTCREATE) {
  407                 FUTEXES_UNLOCK;
  408                 umtx_key_release(&key);
  409                 LIN_SDT_PROBE1(futex, futex_get0, null, uaddr);
  410                 LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr);
  411 
  412                 LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  413                 return (0);
  414         }
  415 
  416         if (tmpf == NULL) {
  417                 FUTEXES_UNLOCK;
  418                 tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO);
  419                 tmpf->f_uaddr = uaddr;
  420                 tmpf->f_key = key;
  421                 tmpf->f_refcount = 1;
  422                 tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY;
  423                 FUTEX_INIT(tmpf);
  424                 TAILQ_INIT(&tmpf->f_waiting_proc);
  425 
  426                 /*
  427                  * Lock the new futex before an insert into the futex_list
  428                  * to prevent futex usage by other.
  429                  */
  430                 if ((flags & FUTEX_DONTLOCK) == 0)
  431                         futex_lock(tmpf);
  432                 goto retry;
  433         }
  434 
  435         LIST_INSERT_HEAD(&futex_list, tmpf, f_list);
  436         FUTEXES_UNLOCK;
  437 
  438         LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount,
  439             tmpf->f_key.shared);
  440         LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new",
  441             uaddr, tmpf->f_refcount, tmpf->f_key.shared);
  442         *newf = tmpf;
  443 
  444         LIN_SDT_PROBE1(futex, futex_get0, return, 0);
  445         return (0);
  446 }
  447 
  448 static int
  449 futex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f,
  450     uint32_t flags)
  451 {
  452         int error;
  453 
  454         LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f);
  455 
  456         if (flags & FUTEX_CREATE_WP) {
  457                 *wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK);
  458                 (*wp)->wp_flags = 0;
  459         }
  460         error = futex_get0(uaddr, f, flags);
  461         if (error) {
  462                 LIN_SDT_PROBE0(futex, futex_get, error);
  463 
  464                 if (flags & FUTEX_CREATE_WP)
  465                         free(*wp, M_FUTEX_WP);
  466 
  467                 LIN_SDT_PROBE1(futex, futex_get, return, error);
  468                 return (error);
  469         }
  470         if (flags & FUTEX_CREATE_WP) {
  471                 TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list);
  472                 (*wp)->wp_futex = *f;
  473         }
  474 
  475         LIN_SDT_PROBE1(futex, futex_get, return, error);
  476         return (error);
  477 }
  478 
  479 static inline void
  480 futex_lock(struct futex *f)
  481 {
  482 
  483         LINUX_CTR3(sys_futex, "futex_lock uaddr %p ref %d shared %d",
  484             f->f_uaddr, f->f_refcount, f->f_key.shared);
  485         FUTEX_ASSERT_UNLOCKED(f);
  486         FUTEX_LOCK(f);
  487 }
  488 
  489 static inline void
  490 futex_unlock(struct futex *f)
  491 {
  492 
  493         LINUX_CTR3(sys_futex, "futex_unlock uaddr %p ref %d shared %d",
  494             f->f_uaddr, f->f_refcount, f->f_key.shared);
  495         FUTEX_ASSERT_LOCKED(f);
  496         FUTEX_UNLOCK(f);
  497 }
  498 
  499 static int
  500 futex_sleep(struct futex *f, struct waiting_proc *wp, struct timespec *ts)
  501 {
  502         struct timespec uts;
  503         sbintime_t sbt, prec, tmp;
  504         time_t over;
  505         int error;
  506 
  507         FUTEX_ASSERT_LOCKED(f);
  508         if (ts != NULL) {
  509                 uts = *ts;
  510                 if (uts.tv_sec > INT32_MAX / 2) {
  511                         over = uts.tv_sec - INT32_MAX / 2;
  512                         uts.tv_sec -= over;
  513                 }
  514                 tmp = tstosbt(uts);
  515                 if (TIMESEL(&sbt, tmp))
  516                         sbt += tc_tick_sbt;
  517                 sbt += tmp;
  518                 prec = tmp;
  519                 prec >>= tc_precexp;
  520         } else {
  521                 sbt = 0;
  522                 prec = 0;
  523         }
  524         LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, sbt);
  525         LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %ld ref %d",
  526             f->f_uaddr, wp, sbt, f->f_refcount);
  527 
  528         error = msleep_sbt(wp, &f->f_lck, PCATCH, "futex", sbt, prec, C_ABSOLUTE);
  529         if (wp->wp_flags & FUTEX_WP_REQUEUED) {
  530                 KASSERT(f != wp->wp_futex, ("futex != wp_futex"));
  531 
  532                 if (error) {
  533                         LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error,
  534                             f->f_uaddr, wp, wp->wp_futex->f_uaddr,
  535                             wp->wp_futex->f_refcount);
  536                 }
  537 
  538                 LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp"
  539                     " %p requeued uaddr %p ref %d",
  540                     error, f->f_uaddr, wp, wp->wp_futex->f_uaddr,
  541                     wp->wp_futex->f_refcount);
  542                 futex_put(f, NULL);
  543                 f = wp->wp_futex;
  544                 futex_lock(f);
  545         } else {
  546                 if (error) {
  547                         LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error,
  548                             f->f_uaddr, wp);
  549                 }
  550                 LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p",
  551                     error, f->f_uaddr, wp);
  552         }
  553 
  554         futex_put(f, wp);
  555 
  556         LIN_SDT_PROBE1(futex, futex_sleep, return, error);
  557         return (error);
  558 }
  559 
  560 static int
  561 futex_wake(struct futex *f, int n, uint32_t bitset)
  562 {
  563         struct waiting_proc *wp, *wpt;
  564         int count = 0;
  565 
  566         LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset);
  567 
  568         if (bitset == 0) {
  569                 LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL);
  570                 return (EINVAL);
  571         }
  572 
  573         FUTEX_ASSERT_LOCKED(f);
  574         TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
  575                 LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp,
  576                     f->f_refcount);
  577                 LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d",
  578                     f->f_uaddr, wp, f->f_refcount);
  579                 /*
  580                  * Unless we find a matching bit in
  581                  * the bitset, continue searching.
  582                  */
  583                 if (!(wp->wp_futex->f_bitset & bitset))
  584                         continue;
  585 
  586                 wp->wp_flags |= FUTEX_WP_REMOVED;
  587                 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  588                 LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp);
  589                 wakeup_one(wp);
  590                 if (++count == n)
  591                         break;
  592         }
  593 
  594         LIN_SDT_PROBE1(futex, futex_wake, return, count);
  595         return (count);
  596 }
  597 
  598 static int
  599 futex_requeue(struct futex *f, int nrwake, struct futex *f2,
  600     int nrrequeue)
  601 {
  602         struct waiting_proc *wp, *wpt;
  603         int count = 0;
  604 
  605         LIN_SDT_PROBE4(futex, futex_requeue, entry, f, nrwake, f2, nrrequeue);
  606 
  607         FUTEX_ASSERT_LOCKED(f);
  608         FUTEX_ASSERT_LOCKED(f2);
  609 
  610         TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
  611                 if (++count <= nrwake) {
  612                         LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p",
  613                             f->f_uaddr, wp);
  614                         wp->wp_flags |= FUTEX_WP_REMOVED;
  615                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  616                         LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp);
  617                         wakeup_one(wp);
  618                 } else {
  619                         LIN_SDT_PROBE3(futex, futex_requeue, requeue,
  620                             f->f_uaddr, wp, f2->f_uaddr);
  621                         LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p",
  622                             f->f_uaddr, wp, f2->f_uaddr);
  623                         wp->wp_flags |= FUTEX_WP_REQUEUED;
  624                         /* Move wp to wp_list of f2 futex */
  625                         TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
  626                         TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list);
  627 
  628                         /*
  629                          * Thread which sleeps on wp after waking should
  630                          * acquire f2 lock, so increment refcount of f2 to
  631                          * prevent it from premature deallocation.
  632                          */
  633                         wp->wp_futex = f2;
  634                         FUTEXES_LOCK;
  635                         ++f2->f_refcount;
  636                         FUTEXES_UNLOCK;
  637                         if (count - nrwake >= nrrequeue)
  638                                 break;
  639                 }
  640         }
  641 
  642         LIN_SDT_PROBE1(futex, futex_requeue, return, count);
  643         return (count);
  644 }
  645 
  646 static int
  647 futex_wait(struct futex *f, struct waiting_proc *wp, struct timespec *ts,
  648     uint32_t bitset)
  649 {
  650         int error;
  651 
  652         LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, ts, bitset);
  653 
  654         if (bitset == 0) {
  655                 LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL);
  656                 return (EINVAL);
  657         }
  658 
  659         f->f_bitset = bitset;
  660         error = futex_sleep(f, wp, ts);
  661         if (error)
  662                 LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error);
  663         if (error == EWOULDBLOCK)
  664                 error = ETIMEDOUT;
  665 
  666         LIN_SDT_PROBE1(futex, futex_wait, return, error);
  667         return (error);
  668 }
  669 
  670 static int
  671 futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr)
  672 {
  673         int op = (encoded_op >> 28) & 7;
  674         int cmp = (encoded_op >> 24) & 15;
  675         int oparg = (encoded_op << 8) >> 20;
  676         int cmparg = (encoded_op << 20) >> 20;
  677         int oldval = 0, ret;
  678 
  679         LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr);
  680 
  681         if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
  682                 oparg = 1 << oparg;
  683 
  684         LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg,
  685             cmparg);
  686 
  687         /* XXX: Linux verifies access here and returns EFAULT */
  688         LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check);
  689 
  690         switch (op) {
  691         case FUTEX_OP_SET:
  692                 ret = futex_xchgl(oparg, uaddr, &oldval);
  693                 break;
  694         case FUTEX_OP_ADD:
  695                 ret = futex_addl(oparg, uaddr, &oldval);
  696                 break;
  697         case FUTEX_OP_OR:
  698                 ret = futex_orl(oparg, uaddr, &oldval);
  699                 break;
  700         case FUTEX_OP_ANDN:
  701                 ret = futex_andl(~oparg, uaddr, &oldval);
  702                 break;
  703         case FUTEX_OP_XOR:
  704                 ret = futex_xorl(oparg, uaddr, &oldval);
  705                 break;
  706         default:
  707                 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op);
  708                 ret = -ENOSYS;
  709                 break;
  710         }
  711 
  712         if (ret) {
  713                 LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
  714                 return (ret);
  715         }
  716 
  717         switch (cmp) {
  718         case FUTEX_OP_CMP_EQ:
  719                 ret = (oldval == cmparg);
  720                 break;
  721         case FUTEX_OP_CMP_NE:
  722                 ret = (oldval != cmparg);
  723                 break;
  724         case FUTEX_OP_CMP_LT:
  725                 ret = (oldval < cmparg);
  726                 break;
  727         case FUTEX_OP_CMP_GE:
  728                 ret = (oldval >= cmparg);
  729                 break;
  730         case FUTEX_OP_CMP_LE:
  731                 ret = (oldval <= cmparg);
  732                 break;
  733         case FUTEX_OP_CMP_GT:
  734                 ret = (oldval > cmparg);
  735                 break;
  736         default:
  737                 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp);
  738                 ret = -ENOSYS;
  739         }
  740 
  741         LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
  742         return (ret);
  743 }
  744 
  745 int
  746 linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
  747 {
  748         int clockrt, nrwake, nrrequeue, op_ret, ret;
  749         struct linux_pemuldata *pem;
  750         struct waiting_proc *wp;
  751         struct futex *f, *f2;
  752         struct timespec uts, *ts;
  753         int error, save;
  754         uint32_t flags, val;
  755 
  756         LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args);
  757 
  758         if (args->op & LINUX_FUTEX_PRIVATE_FLAG) {
  759                 flags = 0;
  760                 args->op &= ~LINUX_FUTEX_PRIVATE_FLAG;
  761         } else
  762                 flags = FUTEX_SHARED;
  763 
  764         /*
  765          * Currently support for switching between CLOCK_MONOTONIC and
  766          * CLOCK_REALTIME is not present. However Linux forbids the use of
  767          * FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and
  768          * FUTEX_WAIT_REQUEUE_PI.
  769          */
  770         clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME;
  771         args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME;
  772         if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET &&
  773                 args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) {
  774                 LIN_SDT_PROBE0(futex, linux_sys_futex,
  775                     unimplemented_clockswitch);
  776                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
  777                 return (ENOSYS);
  778         }
  779 
  780         error = 0;
  781         f = f2 = NULL;
  782 
  783         switch (args->op) {
  784         case LINUX_FUTEX_WAIT:
  785                 args->val3 = FUTEX_BITSET_MATCH_ANY;
  786                 /* FALLTHROUGH */
  787 
  788         case LINUX_FUTEX_WAIT_BITSET:
  789                 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr,
  790                     args->val, args->val3);
  791                 LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x",
  792                     args->uaddr, args->val, args->val3);
  793 
  794                 if (args->timeout != NULL) {
  795                         error = futex_copyin_timeout(args->op, args->timeout,
  796                             clockrt, &uts);
  797                         if (error) {
  798                                 LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  799                                     error);
  800                                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  801                                 return (error);
  802                         }
  803                         ts = &uts;
  804                 } else
  805                         ts = NULL;
  806 
  807 retry0:
  808                 error = futex_get(args->uaddr, &wp, &f,
  809                     flags | FUTEX_CREATE_WP);
  810                 if (error) {
  811                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  812                         return (error);
  813                 }
  814 
  815                 error = copyin_nofault(args->uaddr, &val, sizeof(val));
  816                 if (error) {
  817                         futex_put(f, wp);
  818                         error = copyin(args->uaddr, &val, sizeof(val));
  819                         if (error == 0)
  820                                 goto retry0;
  821                         LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  822                             error);
  823                         LINUX_CTR1(sys_futex, "WAIT copyin failed %d",
  824                             error);
  825                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  826                         return (error);
  827                 }
  828                 if (val != args->val) {
  829                         LIN_SDT_PROBE4(futex, linux_sys_futex,
  830                             debug_wait_value_neq, args->uaddr, args->val, val,
  831                             args->val3);
  832                         LINUX_CTR3(sys_futex,
  833                             "WAIT uaddr %p val 0x%x != uval 0x%x",
  834                             args->uaddr, args->val, val);
  835                         futex_put(f, wp);
  836 
  837                         LIN_SDT_PROBE1(futex, linux_sys_futex, return,
  838                             EWOULDBLOCK);
  839                         return (EWOULDBLOCK);
  840                 }
  841 
  842                 error = futex_wait(f, wp, ts, args->val3);
  843                 break;
  844 
  845         case LINUX_FUTEX_WAKE:
  846                 args->val3 = FUTEX_BITSET_MATCH_ANY;
  847                 /* FALLTHROUGH */
  848 
  849         case LINUX_FUTEX_WAKE_BITSET:
  850                 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr,
  851                     args->val, args->val3);
  852                 LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x",
  853                     args->uaddr, args->val, args->val3);
  854 
  855                 error = futex_get(args->uaddr, NULL, &f,
  856                     flags | FUTEX_DONTCREATE);
  857                 if (error) {
  858                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  859                         return (error);
  860                 }
  861 
  862                 if (f == NULL) {
  863                         td->td_retval[0] = 0;
  864 
  865                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  866                         return (error);
  867                 }
  868                 td->td_retval[0] = futex_wake(f, args->val, args->val3);
  869                 futex_put(f, NULL);
  870                 break;
  871 
  872         case LINUX_FUTEX_CMP_REQUEUE:
  873                 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue,
  874                     args->uaddr, args->val, args->val3, args->uaddr2,
  875                     args->timeout);
  876                 LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p "
  877                     "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x",
  878                     args->uaddr, args->val, args->val3, args->uaddr2,
  879                     args->timeout);
  880 
  881                 /*
  882                  * Linux allows this, we would not, it is an incorrect
  883                  * usage of declared ABI, so return EINVAL.
  884                  */
  885                 if (args->uaddr == args->uaddr2) {
  886                         LIN_SDT_PROBE0(futex, linux_sys_futex,
  887                             invalid_cmp_requeue_use);
  888                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
  889                         return (EINVAL);
  890                 }
  891 
  892                 nrrequeue = (int)(unsigned long)args->timeout;
  893                 nrwake = args->val;
  894                 /*
  895                  * Sanity check to prevent signed integer overflow,
  896                  * see Linux CVE-2018-6927
  897                  */
  898                 if (nrwake < 0 || nrrequeue < 0)
  899                         return (EINVAL);
  900 
  901 retry1:
  902                 error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK);
  903                 if (error) {
  904                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  905                         return (error);
  906                 }
  907 
  908                 /*
  909                  * To avoid deadlocks return EINVAL if second futex
  910                  * exists at this time.
  911                  *
  912                  * Glibc fall back to FUTEX_WAKE in case of any error
  913                  * returned by FUTEX_CMP_REQUEUE.
  914                  */
  915                 error = futex_get(args->uaddr2, NULL, &f2,
  916                     flags | FUTEX_DONTEXISTS | FUTEX_DONTLOCK);
  917                 if (error) {
  918                         futex_put(f, NULL);
  919 
  920                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  921                         return (error);
  922                 }
  923                 futex_lock(f);
  924                 futex_lock(f2);
  925                 error = copyin_nofault(args->uaddr, &val, sizeof(val));
  926                 if (error) {
  927                         futex_put(f2, NULL);
  928                         futex_put(f, NULL);
  929                         error = copyin(args->uaddr, &val, sizeof(val));
  930                         if (error == 0)
  931                                 goto retry1;
  932                         LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
  933                             error);
  934                         LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d",
  935                             error);
  936                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  937                         return (error);
  938                 }
  939                 if (val != args->val3) {
  940                         LIN_SDT_PROBE2(futex, linux_sys_futex,
  941                             debug_cmp_requeue_value_neq, args->val, val);
  942                         LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x",
  943                             args->val, val);
  944                         futex_put(f2, NULL);
  945                         futex_put(f, NULL);
  946 
  947                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN);
  948                         return (EAGAIN);
  949                 }
  950 
  951                 td->td_retval[0] = futex_requeue(f, nrwake, f2, nrrequeue);
  952                 futex_put(f2, NULL);
  953                 futex_put(f, NULL);
  954                 break;
  955 
  956         case LINUX_FUTEX_WAKE_OP:
  957                 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op,
  958                     args->uaddr, args->op, args->val, args->uaddr2, args->val3);
  959                 LINUX_CTR5(sys_futex, "WAKE_OP "
  960                     "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x",
  961                     args->uaddr, args->val, args->uaddr2, args->val3,
  962                     args->timeout);
  963 
  964                 if (args->uaddr == args->uaddr2) {
  965                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
  966                         return (EINVAL);
  967                 }
  968 
  969 retry2:
  970                 error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK);
  971                 if (error) {
  972                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  973                         return (error);
  974                 }
  975 
  976                 error = futex_get(args->uaddr2, NULL, &f2, flags | FUTEX_DONTLOCK);
  977                 if (error) {
  978                         futex_put(f, NULL);
  979 
  980                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
  981                         return (error);
  982                 }
  983                 futex_lock(f);
  984                 futex_lock(f2);
  985 
  986                 /*
  987                  * This function returns positive number as results and
  988                  * negative as errors
  989                  */
  990                 save = vm_fault_disable_pagefaults();
  991                 op_ret = futex_atomic_op(td, args->val3, args->uaddr2);
  992                 vm_fault_enable_pagefaults(save);
  993 
  994                 LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x",
  995                     args->uaddr, op_ret);
  996 
  997                 if (op_ret < 0) {
  998                         if (f2 != NULL)
  999                                 futex_put(f2, NULL);
 1000                         futex_put(f, NULL);
 1001                         error = copyin(args->uaddr2, &val, sizeof(val));
 1002                         if (error == 0)
 1003                                 goto retry2;
 1004                         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
 1005                         return (error);
 1006                 }
 1007 
 1008                 ret = futex_wake(f, args->val, args->val3);
 1009 
 1010                 if (op_ret > 0) {
 1011                         op_ret = 0;
 1012                         nrwake = (int)(unsigned long)args->timeout;
 1013 
 1014                         if (f2 != NULL)
 1015                                 op_ret += futex_wake(f2, nrwake, args->val3);
 1016                         else
 1017                                 op_ret += futex_wake(f, nrwake, args->val3);
 1018                         ret += op_ret;
 1019 
 1020                 }
 1021                 if (f2 != NULL)
 1022                         futex_put(f2, NULL);
 1023                 futex_put(f, NULL);
 1024                 td->td_retval[0] = ret;
 1025                 break;
 1026 
 1027         case LINUX_FUTEX_LOCK_PI:
 1028                 /* not yet implemented */
 1029                 pem = pem_find(td->td_proc);
 1030                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1031                         linux_msg(td,
 1032                                   "linux_sys_futex: "
 1033                                   "unsupported futex_pi op\n");
 1034                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1035                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1036                             unimplemented_lock_pi);
 1037                 }
 1038                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1039                 return (ENOSYS);
 1040 
 1041         case LINUX_FUTEX_UNLOCK_PI:
 1042                 /* not yet implemented */
 1043                 pem = pem_find(td->td_proc);
 1044                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1045                         linux_msg(td,
 1046                                   "linux_sys_futex: "
 1047                                   "unsupported futex_pi op\n");
 1048                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1049                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1050                             unimplemented_unlock_pi);
 1051                 }
 1052                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1053                 return (ENOSYS);
 1054 
 1055         case LINUX_FUTEX_TRYLOCK_PI:
 1056                 /* not yet implemented */
 1057                 pem = pem_find(td->td_proc);
 1058                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1059                         linux_msg(td,
 1060                                   "linux_sys_futex: "
 1061                                   "unsupported futex_pi op\n");
 1062                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1063                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1064                             unimplemented_trylock_pi);
 1065                 }
 1066                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1067                 return (ENOSYS);
 1068 
 1069         case LINUX_FUTEX_REQUEUE:
 1070                 /*
 1071                  * Glibc does not use this operation since version 2.3.3,
 1072                  * as it is racy and replaced by FUTEX_CMP_REQUEUE operation.
 1073                  * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when
 1074                  * FUTEX_REQUEUE returned EINVAL.
 1075                  */
 1076                 pem = pem_find(td->td_proc);
 1077                 if ((pem->flags & LINUX_XDEPR_REQUEUEOP) == 0) {
 1078                         linux_msg(td,
 1079                                   "linux_sys_futex: "
 1080                                   "unsupported futex_requeue op\n");
 1081                         pem->flags |= LINUX_XDEPR_REQUEUEOP;
 1082                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1083                             deprecated_requeue);
 1084                 }
 1085 
 1086                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
 1087                 return (EINVAL);
 1088 
 1089         case LINUX_FUTEX_WAIT_REQUEUE_PI:
 1090                 /* not yet implemented */
 1091                 pem = pem_find(td->td_proc);
 1092                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1093                         linux_msg(td,
 1094                                   "linux_sys_futex: "
 1095                                   "unsupported futex_pi op\n");
 1096                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1097                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1098                             unimplemented_wait_requeue_pi);
 1099                 }
 1100                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1101                 return (ENOSYS);
 1102 
 1103         case LINUX_FUTEX_CMP_REQUEUE_PI:
 1104                 /* not yet implemented */
 1105                 pem = pem_find(td->td_proc);
 1106                 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
 1107                         linux_msg(td,
 1108                                   "linux_sys_futex: "
 1109                                   "unsupported futex_pi op\n");
 1110                         pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
 1111                         LIN_SDT_PROBE0(futex, linux_sys_futex,
 1112                             unimplemented_cmp_requeue_pi);
 1113                 }
 1114                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1115                 return (ENOSYS);
 1116 
 1117         default:
 1118                 linux_msg(td,
 1119                           "linux_sys_futex: unknown op %d\n", args->op);
 1120                 LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation,
 1121                     args->op);
 1122                 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
 1123                 return (ENOSYS);
 1124         }
 1125 
 1126         LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
 1127         return (error);
 1128 }
 1129 
 1130 int
 1131 linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args)
 1132 {
 1133         struct linux_emuldata *em;
 1134 
 1135         LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args);
 1136 
 1137         if (args->len != sizeof(struct linux_robust_list_head)) {
 1138                 LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error);
 1139                 LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL);
 1140                 return (EINVAL);
 1141         }
 1142 
 1143         em = em_find(td);
 1144         em->robust_futexes = args->head;
 1145 
 1146         LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0);
 1147         return (0);
 1148 }
 1149 
 1150 int
 1151 linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args)
 1152 {
 1153         struct linux_emuldata *em;
 1154         struct linux_robust_list_head *head;
 1155         l_size_t len = sizeof(struct linux_robust_list_head);
 1156         struct thread *td2;
 1157         int error = 0;
 1158 
 1159         LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args);
 1160 
 1161         if (!args->pid) {
 1162                 em = em_find(td);
 1163                 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
 1164                 head = em->robust_futexes;
 1165         } else {
 1166                 td2 = tdfind(args->pid, -1);
 1167                 if (td2 == NULL) {
 1168                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1169                             ESRCH);
 1170                         return (ESRCH);
 1171                 }
 1172                 if (SV_PROC_ABI(td2->td_proc) != SV_ABI_LINUX) {
 1173                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1174                             EPERM);
 1175                         PROC_UNLOCK(td2->td_proc);
 1176                         return (EPERM);
 1177                 }
 1178 
 1179                 em = em_find(td2);
 1180                 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
 1181                 /* XXX: ptrace? */
 1182                 if (priv_check(td, PRIV_CRED_SETUID) ||
 1183                     priv_check(td, PRIV_CRED_SETEUID) ||
 1184                     p_candebug(td, td2->td_proc)) {
 1185                         PROC_UNLOCK(td2->td_proc);
 1186 
 1187                         LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
 1188                             EPERM);
 1189                         return (EPERM);
 1190                 }
 1191                 head = em->robust_futexes;
 1192 
 1193                 PROC_UNLOCK(td2->td_proc);
 1194         }
 1195 
 1196         error = copyout(&len, args->len, sizeof(l_size_t));
 1197         if (error) {
 1198                 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
 1199                     error);
 1200                 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT);
 1201                 return (EFAULT);
 1202         }
 1203 
 1204         error = copyout(&head, args->head, sizeof(head));
 1205         if (error) {
 1206                 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
 1207                     error);
 1208         }
 1209 
 1210         LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error);
 1211         return (error);
 1212 }
 1213 
 1214 static int
 1215 handle_futex_death(struct linux_emuldata *em, uint32_t *uaddr,
 1216     unsigned int pi)
 1217 {
 1218         uint32_t uval, nval, mval;
 1219         struct futex *f;
 1220         int error;
 1221 
 1222         LIN_SDT_PROBE3(futex, handle_futex_death, entry, em, uaddr, pi);
 1223 
 1224 retry:
 1225         error = copyin(uaddr, &uval, 4);
 1226         if (error) {
 1227                 LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error);
 1228                 LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT);
 1229                 return (EFAULT);
 1230         }
 1231         if ((uval & FUTEX_TID_MASK) == em->em_tid) {
 1232                 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
 1233                 nval = casuword32(uaddr, uval, mval);
 1234 
 1235                 if (nval == -1) {
 1236                         LIN_SDT_PROBE1(futex, handle_futex_death, return,
 1237                             EFAULT);
 1238                         return (EFAULT);
 1239                 }
 1240 
 1241                 if (nval != uval)
 1242                         goto retry;
 1243 
 1244                 if (!pi && (uval & FUTEX_WAITERS)) {
 1245                         error = futex_get(uaddr, NULL, &f,
 1246                             FUTEX_DONTCREATE | FUTEX_SHARED);
 1247                         if (error) {
 1248                                 LIN_SDT_PROBE1(futex, handle_futex_death,
 1249                                     return, error);
 1250                                 return (error);
 1251                         }
 1252                         if (f != NULL) {
 1253                                 futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY);
 1254                                 futex_put(f, NULL);
 1255                         }
 1256                 }
 1257         }
 1258 
 1259         LIN_SDT_PROBE1(futex, handle_futex_death, return, 0);
 1260         return (0);
 1261 }
 1262 
 1263 static int
 1264 fetch_robust_entry(struct linux_robust_list **entry,
 1265     struct linux_robust_list **head, unsigned int *pi)
 1266 {
 1267         l_ulong uentry;
 1268         int error;
 1269 
 1270         LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi);
 1271 
 1272         error = copyin((const void *)head, &uentry, sizeof(l_ulong));
 1273         if (error) {
 1274                 LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error);
 1275                 LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT);
 1276                 return (EFAULT);
 1277         }
 1278 
 1279         *entry = (void *)(uentry & ~1UL);
 1280         *pi = uentry & 1;
 1281 
 1282         LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0);
 1283         return (0);
 1284 }
 1285 
 1286 /* This walks the list of robust futexes releasing them. */
 1287 void
 1288 release_futexes(struct thread *td, struct linux_emuldata *em)
 1289 {
 1290         struct linux_robust_list_head *head = NULL;
 1291         struct linux_robust_list *entry, *next_entry, *pending;
 1292         unsigned int limit = 2048, pi, next_pi, pip;
 1293         l_long futex_offset;
 1294         int rc, error;
 1295 
 1296         LIN_SDT_PROBE2(futex, release_futexes, entry, td, em);
 1297 
 1298         head = em->robust_futexes;
 1299 
 1300         if (head == NULL) {
 1301                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1302                 return;
 1303         }
 1304 
 1305         if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) {
 1306                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1307                 return;
 1308         }
 1309 
 1310         error = copyin(&head->futex_offset, &futex_offset,
 1311             sizeof(futex_offset));
 1312         if (error) {
 1313                 LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error);
 1314                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1315                 return;
 1316         }
 1317 
 1318         if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) {
 1319                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1320                 return;
 1321         }
 1322 
 1323         while (entry != &head->list) {
 1324                 rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi);
 1325 
 1326                 if (entry != pending)
 1327                         if (handle_futex_death(em,
 1328                             (uint32_t *)((caddr_t)entry + futex_offset), pi)) {
 1329                                 LIN_SDT_PROBE0(futex, release_futexes, return);
 1330                                 return;
 1331                         }
 1332                 if (rc) {
 1333                         LIN_SDT_PROBE0(futex, release_futexes, return);
 1334                         return;
 1335                 }
 1336 
 1337                 entry = next_entry;
 1338                 pi = next_pi;
 1339 
 1340                 if (!--limit)
 1341                         break;
 1342 
 1343                 sched_relinquish(curthread);
 1344         }
 1345 
 1346         if (pending)
 1347                 handle_futex_death(em, (uint32_t *)((caddr_t)pending + futex_offset), pip);
 1348 
 1349         LIN_SDT_PROBE0(futex, release_futexes, return);
 1350 }

Cache object: 5e2c42a620ac70d28e073197c0b5c471


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.