The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/sys/stdatomic.h

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
    3  *                    David Chisnall <theraven@FreeBSD.org>
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  *
   27  * $FreeBSD: releng/10.3/sys/sys/stdatomic.h 264496 2014-04-15 09:41:52Z tijl $
   28  */
   29 
   30 #ifndef _STDATOMIC_H_
   31 #define _STDATOMIC_H_
   32 
   33 #include <sys/cdefs.h>
   34 #include <sys/_types.h>
   35 
   36 #if __has_extension(c_atomic) || __has_extension(cxx_atomic)
   37 #define __CLANG_ATOMICS
   38 #elif __GNUC_PREREQ__(4, 7)
   39 #define __GNUC_ATOMICS
   40 #elif defined(__GNUC__)
   41 #define __SYNC_ATOMICS
   42 #else
   43 #error "stdatomic.h does not support your compiler"
   44 #endif
   45 
   46 /*
   47  * 7.17.1 Atomic lock-free macros.
   48  */
   49 
   50 #ifdef __GCC_ATOMIC_BOOL_LOCK_FREE
   51 #define ATOMIC_BOOL_LOCK_FREE           __GCC_ATOMIC_BOOL_LOCK_FREE
   52 #endif
   53 #ifdef __GCC_ATOMIC_CHAR_LOCK_FREE
   54 #define ATOMIC_CHAR_LOCK_FREE           __GCC_ATOMIC_CHAR_LOCK_FREE
   55 #endif
   56 #ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE
   57 #define ATOMIC_CHAR16_T_LOCK_FREE       __GCC_ATOMIC_CHAR16_T_LOCK_FREE
   58 #endif
   59 #ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE
   60 #define ATOMIC_CHAR32_T_LOCK_FREE       __GCC_ATOMIC_CHAR32_T_LOCK_FREE
   61 #endif
   62 #ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE
   63 #define ATOMIC_WCHAR_T_LOCK_FREE        __GCC_ATOMIC_WCHAR_T_LOCK_FREE
   64 #endif
   65 #ifdef __GCC_ATOMIC_SHORT_LOCK_FREE
   66 #define ATOMIC_SHORT_LOCK_FREE          __GCC_ATOMIC_SHORT_LOCK_FREE
   67 #endif
   68 #ifdef __GCC_ATOMIC_INT_LOCK_FREE
   69 #define ATOMIC_INT_LOCK_FREE            __GCC_ATOMIC_INT_LOCK_FREE
   70 #endif
   71 #ifdef __GCC_ATOMIC_LONG_LOCK_FREE
   72 #define ATOMIC_LONG_LOCK_FREE           __GCC_ATOMIC_LONG_LOCK_FREE
   73 #endif
   74 #ifdef __GCC_ATOMIC_LLONG_LOCK_FREE
   75 #define ATOMIC_LLONG_LOCK_FREE          __GCC_ATOMIC_LLONG_LOCK_FREE
   76 #endif
   77 #ifdef __GCC_ATOMIC_POINTER_LOCK_FREE
   78 #define ATOMIC_POINTER_LOCK_FREE        __GCC_ATOMIC_POINTER_LOCK_FREE
   79 #endif
   80 
   81 /*
   82  * 7.17.2 Initialization.
   83  */
   84 
   85 #if defined(__CLANG_ATOMICS)
   86 #define ATOMIC_VAR_INIT(value)          (value)
   87 #define atomic_init(obj, value)         __c11_atomic_init(obj, value)
   88 #else
   89 #define ATOMIC_VAR_INIT(value)          { .__val = (value) }
   90 #define atomic_init(obj, value)         ((void)((obj)->__val = (value)))
   91 #endif
   92 
   93 /*
   94  * Clang and recent GCC both provide predefined macros for the memory
   95  * orderings.  If we are using a compiler that doesn't define them, use the
   96  * clang values - these will be ignored in the fallback path.
   97  */
   98 
   99 #ifndef __ATOMIC_RELAXED
  100 #define __ATOMIC_RELAXED                0
  101 #endif
  102 #ifndef __ATOMIC_CONSUME
  103 #define __ATOMIC_CONSUME                1
  104 #endif
  105 #ifndef __ATOMIC_ACQUIRE
  106 #define __ATOMIC_ACQUIRE                2
  107 #endif
  108 #ifndef __ATOMIC_RELEASE
  109 #define __ATOMIC_RELEASE                3
  110 #endif
  111 #ifndef __ATOMIC_ACQ_REL
  112 #define __ATOMIC_ACQ_REL                4
  113 #endif
  114 #ifndef __ATOMIC_SEQ_CST
  115 #define __ATOMIC_SEQ_CST                5
  116 #endif
  117 
  118 /*
  119  * 7.17.3 Order and consistency.
  120  *
  121  * The memory_order_* constants that denote the barrier behaviour of the
  122  * atomic operations.
  123  */
  124 
  125 typedef enum {
  126         memory_order_relaxed = __ATOMIC_RELAXED,
  127         memory_order_consume = __ATOMIC_CONSUME,
  128         memory_order_acquire = __ATOMIC_ACQUIRE,
  129         memory_order_release = __ATOMIC_RELEASE,
  130         memory_order_acq_rel = __ATOMIC_ACQ_REL,
  131         memory_order_seq_cst = __ATOMIC_SEQ_CST
  132 } memory_order;
  133 
  134 /*
  135  * 7.17.4 Fences.
  136  */
  137 
  138 static __inline void
  139 atomic_thread_fence(memory_order __order __unused)
  140 {
  141 
  142 #ifdef __CLANG_ATOMICS
  143         __c11_atomic_thread_fence(__order);
  144 #elif defined(__GNUC_ATOMICS)
  145         __atomic_thread_fence(__order);
  146 #else
  147         __sync_synchronize();
  148 #endif
  149 }
  150 
  151 static __inline void
  152 atomic_signal_fence(memory_order __order __unused)
  153 {
  154 
  155 #ifdef __CLANG_ATOMICS
  156         __c11_atomic_signal_fence(__order);
  157 #elif defined(__GNUC_ATOMICS)
  158         __atomic_signal_fence(__order);
  159 #else
  160         __asm volatile ("" ::: "memory");
  161 #endif
  162 }
  163 
  164 /*
  165  * 7.17.5 Lock-free property.
  166  */
  167 
  168 #if defined(_KERNEL)
  169 /* Atomics in kernelspace are always lock-free. */
  170 #define atomic_is_lock_free(obj) \
  171         ((void)(obj), (_Bool)1)
  172 #elif defined(__CLANG_ATOMICS)
  173 #define atomic_is_lock_free(obj) \
  174         __atomic_is_lock_free(sizeof(*(obj)), obj)
  175 #elif defined(__GNUC_ATOMICS)
  176 #define atomic_is_lock_free(obj) \
  177         __atomic_is_lock_free(sizeof((obj)->__val), &(obj)->__val)
  178 #else
  179 #define atomic_is_lock_free(obj) \
  180         ((void)(obj), sizeof((obj)->__val) <= sizeof(void *))
  181 #endif
  182 
  183 /*
  184  * 7.17.6 Atomic integer types.
  185  */
  186 
  187 typedef _Atomic(_Bool)                  atomic_bool;
  188 typedef _Atomic(char)                   atomic_char;
  189 typedef _Atomic(signed char)            atomic_schar;
  190 typedef _Atomic(unsigned char)          atomic_uchar;
  191 typedef _Atomic(short)                  atomic_short;
  192 typedef _Atomic(unsigned short)         atomic_ushort;
  193 typedef _Atomic(int)                    atomic_int;
  194 typedef _Atomic(unsigned int)           atomic_uint;
  195 typedef _Atomic(long)                   atomic_long;
  196 typedef _Atomic(unsigned long)          atomic_ulong;
  197 typedef _Atomic(long long)              atomic_llong;
  198 typedef _Atomic(unsigned long long)     atomic_ullong;
  199 typedef _Atomic(__char16_t)             atomic_char16_t;
  200 typedef _Atomic(__char32_t)             atomic_char32_t;
  201 typedef _Atomic(___wchar_t)             atomic_wchar_t;
  202 typedef _Atomic(__int_least8_t)         atomic_int_least8_t;
  203 typedef _Atomic(__uint_least8_t)        atomic_uint_least8_t;
  204 typedef _Atomic(__int_least16_t)        atomic_int_least16_t;
  205 typedef _Atomic(__uint_least16_t)       atomic_uint_least16_t;
  206 typedef _Atomic(__int_least32_t)        atomic_int_least32_t;
  207 typedef _Atomic(__uint_least32_t)       atomic_uint_least32_t;
  208 typedef _Atomic(__int_least64_t)        atomic_int_least64_t;
  209 typedef _Atomic(__uint_least64_t)       atomic_uint_least64_t;
  210 typedef _Atomic(__int_fast8_t)          atomic_int_fast8_t;
  211 typedef _Atomic(__uint_fast8_t)         atomic_uint_fast8_t;
  212 typedef _Atomic(__int_fast16_t)         atomic_int_fast16_t;
  213 typedef _Atomic(__uint_fast16_t)        atomic_uint_fast16_t;
  214 typedef _Atomic(__int_fast32_t)         atomic_int_fast32_t;
  215 typedef _Atomic(__uint_fast32_t)        atomic_uint_fast32_t;
  216 typedef _Atomic(__int_fast64_t)         atomic_int_fast64_t;
  217 typedef _Atomic(__uint_fast64_t)        atomic_uint_fast64_t;
  218 typedef _Atomic(__intptr_t)             atomic_intptr_t;
  219 typedef _Atomic(__uintptr_t)            atomic_uintptr_t;
  220 typedef _Atomic(__size_t)               atomic_size_t;
  221 typedef _Atomic(__ptrdiff_t)            atomic_ptrdiff_t;
  222 typedef _Atomic(__intmax_t)             atomic_intmax_t;
  223 typedef _Atomic(__uintmax_t)            atomic_uintmax_t;
  224 
  225 /*
  226  * 7.17.7 Operations on atomic types.
  227  */
  228 
  229 /*
  230  * Compiler-specific operations.
  231  */
  232 
  233 #if defined(__CLANG_ATOMICS)
  234 #define atomic_compare_exchange_strong_explicit(object, expected,       \
  235     desired, success, failure)                                          \
  236         __c11_atomic_compare_exchange_strong(object, expected, desired, \
  237             success, failure)
  238 #define atomic_compare_exchange_weak_explicit(object, expected,         \
  239     desired, success, failure)                                          \
  240         __c11_atomic_compare_exchange_weak(object, expected, desired,   \
  241             success, failure)
  242 #define atomic_exchange_explicit(object, desired, order)                \
  243         __c11_atomic_exchange(object, desired, order)
  244 #define atomic_fetch_add_explicit(object, operand, order)               \
  245         __c11_atomic_fetch_add(object, operand, order)
  246 #define atomic_fetch_and_explicit(object, operand, order)               \
  247         __c11_atomic_fetch_and(object, operand, order)
  248 #define atomic_fetch_or_explicit(object, operand, order)                \
  249         __c11_atomic_fetch_or(object, operand, order)
  250 #define atomic_fetch_sub_explicit(object, operand, order)               \
  251         __c11_atomic_fetch_sub(object, operand, order)
  252 #define atomic_fetch_xor_explicit(object, operand, order)               \
  253         __c11_atomic_fetch_xor(object, operand, order)
  254 #define atomic_load_explicit(object, order)                             \
  255         __c11_atomic_load(object, order)
  256 #define atomic_store_explicit(object, desired, order)                   \
  257         __c11_atomic_store(object, desired, order)
  258 #elif defined(__GNUC_ATOMICS)
  259 #define atomic_compare_exchange_strong_explicit(object, expected,       \
  260     desired, success, failure)                                          \
  261         __atomic_compare_exchange_n(&(object)->__val, expected,         \
  262             desired, 0, success, failure)
  263 #define atomic_compare_exchange_weak_explicit(object, expected,         \
  264     desired, success, failure)                                          \
  265         __atomic_compare_exchange_n(&(object)->__val, expected,         \
  266             desired, 1, success, failure)
  267 #define atomic_exchange_explicit(object, desired, order)                \
  268         __atomic_exchange_n(&(object)->__val, desired, order)
  269 #define atomic_fetch_add_explicit(object, operand, order)               \
  270         __atomic_fetch_add(&(object)->__val, operand, order)
  271 #define atomic_fetch_and_explicit(object, operand, order)               \
  272         __atomic_fetch_and(&(object)->__val, operand, order)
  273 #define atomic_fetch_or_explicit(object, operand, order)                \
  274         __atomic_fetch_or(&(object)->__val, operand, order)
  275 #define atomic_fetch_sub_explicit(object, operand, order)               \
  276         __atomic_fetch_sub(&(object)->__val, operand, order)
  277 #define atomic_fetch_xor_explicit(object, operand, order)               \
  278         __atomic_fetch_xor(&(object)->__val, operand, order)
  279 #define atomic_load_explicit(object, order)                             \
  280         __atomic_load_n(&(object)->__val, order)
  281 #define atomic_store_explicit(object, desired, order)                   \
  282         __atomic_store_n(&(object)->__val, desired, order)
  283 #else
  284 #define __atomic_apply_stride(object, operand) \
  285         (((__typeof__((object)->__val))0) + (operand))
  286 #define atomic_compare_exchange_strong_explicit(object, expected,       \
  287     desired, success, failure)  __extension__ ({                        \
  288         __typeof__(expected) __ep = (expected);                         \
  289         __typeof__(*__ep) __e = *__ep;                                  \
  290         (void)(success); (void)(failure);                               \
  291         (_Bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val,  \
  292             __e, desired)) == __e);                                     \
  293 })
  294 #define atomic_compare_exchange_weak_explicit(object, expected,         \
  295     desired, success, failure)                                          \
  296         atomic_compare_exchange_strong_explicit(object, expected,       \
  297                 desired, success, failure)
  298 #if __has_builtin(__sync_swap)
  299 /* Clang provides a full-barrier atomic exchange - use it if available. */
  300 #define atomic_exchange_explicit(object, desired, order)                \
  301         ((void)(order), __sync_swap(&(object)->__val, desired))
  302 #else
  303 /*
  304  * __sync_lock_test_and_set() is only an acquire barrier in theory (although in
  305  * practice it is usually a full barrier) so we need an explicit barrier before
  306  * it.
  307  */
  308 #define atomic_exchange_explicit(object, desired, order)                \
  309 __extension__ ({                                                        \
  310         __typeof__(object) __o = (object);                              \
  311         __typeof__(desired) __d = (desired);                            \
  312         (void)(order);                                                  \
  313         __sync_synchronize();                                           \
  314         __sync_lock_test_and_set(&(__o)->__val, __d);                   \
  315 })
  316 #endif
  317 #define atomic_fetch_add_explicit(object, operand, order)               \
  318         ((void)(order), __sync_fetch_and_add(&(object)->__val,          \
  319             __atomic_apply_stride(object, operand)))
  320 #define atomic_fetch_and_explicit(object, operand, order)               \
  321         ((void)(order), __sync_fetch_and_and(&(object)->__val, operand))
  322 #define atomic_fetch_or_explicit(object, operand, order)                \
  323         ((void)(order), __sync_fetch_and_or(&(object)->__val, operand))
  324 #define atomic_fetch_sub_explicit(object, operand, order)               \
  325         ((void)(order), __sync_fetch_and_sub(&(object)->__val,          \
  326             __atomic_apply_stride(object, operand)))
  327 #define atomic_fetch_xor_explicit(object, operand, order)               \
  328         ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand))
  329 #define atomic_load_explicit(object, order)                             \
  330         ((void)(order), __sync_fetch_and_add(&(object)->__val, 0))
  331 #define atomic_store_explicit(object, desired, order)                   \
  332         ((void)atomic_exchange_explicit(object, desired, order))
  333 #endif
  334 
  335 /*
  336  * Convenience functions.
  337  *
  338  * Don't provide these in kernel space. In kernel space, we should be
  339  * disciplined enough to always provide explicit barriers.
  340  */
  341 
  342 #ifndef _KERNEL
  343 #define atomic_compare_exchange_strong(object, expected, desired)       \
  344         atomic_compare_exchange_strong_explicit(object, expected,       \
  345             desired, memory_order_seq_cst, memory_order_seq_cst)
  346 #define atomic_compare_exchange_weak(object, expected, desired)         \
  347         atomic_compare_exchange_weak_explicit(object, expected,         \
  348             desired, memory_order_seq_cst, memory_order_seq_cst)
  349 #define atomic_exchange(object, desired)                                \
  350         atomic_exchange_explicit(object, desired, memory_order_seq_cst)
  351 #define atomic_fetch_add(object, operand)                               \
  352         atomic_fetch_add_explicit(object, operand, memory_order_seq_cst)
  353 #define atomic_fetch_and(object, operand)                               \
  354         atomic_fetch_and_explicit(object, operand, memory_order_seq_cst)
  355 #define atomic_fetch_or(object, operand)                                \
  356         atomic_fetch_or_explicit(object, operand, memory_order_seq_cst)
  357 #define atomic_fetch_sub(object, operand)                               \
  358         atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst)
  359 #define atomic_fetch_xor(object, operand)                               \
  360         atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst)
  361 #define atomic_load(object)                                             \
  362         atomic_load_explicit(object, memory_order_seq_cst)
  363 #define atomic_store(object, desired)                                   \
  364         atomic_store_explicit(object, desired, memory_order_seq_cst)
  365 #endif /* !_KERNEL */
  366 
  367 /*
  368  * 7.17.8 Atomic flag type and operations.
  369  *
  370  * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some
  371  * kind of compiler built-in type we could use?
  372  */
  373 
  374 typedef struct {
  375         atomic_bool     __flag;
  376 } atomic_flag;
  377 
  378 #define ATOMIC_FLAG_INIT                { ATOMIC_VAR_INIT(0) }
  379 
  380 static __inline _Bool
  381 atomic_flag_test_and_set_explicit(volatile atomic_flag *__object,
  382     memory_order __order)
  383 {
  384         return (atomic_exchange_explicit(&__object->__flag, 1, __order));
  385 }
  386 
  387 static __inline void
  388 atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order)
  389 {
  390 
  391         atomic_store_explicit(&__object->__flag, 0, __order);
  392 }
  393 
  394 #ifndef _KERNEL
  395 static __inline _Bool
  396 atomic_flag_test_and_set(volatile atomic_flag *__object)
  397 {
  398 
  399         return (atomic_flag_test_and_set_explicit(__object,
  400             memory_order_seq_cst));
  401 }
  402 
  403 static __inline void
  404 atomic_flag_clear(volatile atomic_flag *__object)
  405 {
  406 
  407         atomic_flag_clear_explicit(__object, memory_order_seq_cst);
  408 }
  409 #endif /* !_KERNEL */
  410 
  411 #endif /* !_STDATOMIC_H_ */

Cache object: 4643fc52e3a7803444205824253f0b71


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.