Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/mips/mips/stdatomic.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD: releng/11.1/sys/mips/mips/stdatomic.c 274668 2014-11-18 17:06:56Z imp $"); 32 33 #include <sys/stdatomic.h> 34 #include <sys/types.h> 35 36 #if defined(__SYNC_ATOMICS) 37 38 /* 39 * Memory barriers. 40 * 41 * It turns out __sync_synchronize() does not emit any code when used 42 * with GCC 4.2. Implement our own version that does work reliably. 43 * 44 * Although __sync_lock_test_and_set() should only perform an acquire 45 * barrier, make it do a full barrier like the other functions. This 46 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably. 47 */ 48 49 static inline void 50 do_sync(void) 51 { 52 53 __asm volatile ( 54 #if !defined(_KERNEL) || defined(SMP) 55 ".set noreorder\n" 56 "\tsync\n" 57 "\tnop\n" 58 "\tnop\n" 59 "\tnop\n" 60 "\tnop\n" 61 "\tnop\n" 62 "\tnop\n" 63 "\tnop\n" 64 "\tnop\n" 65 ".set reorder\n" 66 #else /* _KERNEL && !SMP */ 67 "" 68 #endif /* !KERNEL || SMP */ 69 : : : "memory"); 70 } 71 72 typedef union { 73 uint8_t v8[4]; 74 uint32_t v32; 75 } reg_t; 76 77 /* 78 * Given a memory address pointing to an 8-bit or 16-bit integer, return 79 * the address of the 32-bit word containing it. 80 */ 81 82 static inline uint32_t * 83 round_to_word(void *ptr) 84 { 85 86 return ((uint32_t *)((intptr_t)ptr & ~3)); 87 } 88 89 /* 90 * Utility functions for loading and storing 8-bit and 16-bit integers 91 * in 32-bit words at an offset corresponding with the location of the 92 * atomic variable. 93 */ 94 95 static inline void 96 put_1(reg_t *r, const uint8_t *offset_ptr, uint8_t val) 97 { 98 size_t offset; 99 100 offset = (intptr_t)offset_ptr & 3; 101 r->v8[offset] = val; 102 } 103 104 static inline uint8_t 105 get_1(const reg_t *r, const uint8_t *offset_ptr) 106 { 107 size_t offset; 108 109 offset = (intptr_t)offset_ptr & 3; 110 return (r->v8[offset]); 111 } 112 113 static inline void 114 put_2(reg_t *r, const uint16_t *offset_ptr, uint16_t val) 115 { 116 size_t offset; 117 union { 118 uint16_t in; 119 uint8_t out[2]; 120 } bytes; 121 122 offset = (intptr_t)offset_ptr & 3; 123 bytes.in = val; 124 r->v8[offset] = bytes.out[0]; 125 r->v8[offset + 1] = bytes.out[1]; 126 } 127 128 static inline uint16_t 129 get_2(const reg_t *r, const uint16_t *offset_ptr) 130 { 131 size_t offset; 132 union { 133 uint8_t in[2]; 134 uint16_t out; 135 } bytes; 136 137 offset = (intptr_t)offset_ptr & 3; 138 bytes.in[0] = r->v8[offset]; 139 bytes.in[1] = r->v8[offset + 1]; 140 return (bytes.out); 141 } 142 143 /* 144 * 8-bit and 16-bit routines. 145 * 146 * These operations are not natively supported by the CPU, so we use 147 * some shifting and bitmasking on top of the 32-bit instructions. 148 */ 149 150 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t) \ 151 uintN_t \ 152 __sync_lock_test_and_set_##N(uintN_t *mem, uintN_t val) \ 153 { \ 154 uint32_t *mem32; \ 155 reg_t val32, negmask, old; \ 156 uint32_t temp; \ 157 \ 158 mem32 = round_to_word(mem); \ 159 val32.v32 = 0x00000000; \ 160 put_##N(&val32, mem, val); \ 161 negmask.v32 = 0xffffffff; \ 162 put_##N(&negmask, mem, 0); \ 163 \ 164 do_sync(); \ 165 __asm volatile ( \ 166 "1:" \ 167 "\tll %0, %5\n" /* Load old value. */ \ 168 "\tand %2, %4, %0\n" /* Remove the old value. */ \ 169 "\tor %2, %3\n" /* Put in the new value. */ \ 170 "\tsc %2, %1\n" /* Attempt to store. */ \ 171 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 172 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 173 : "r" (val32.v32), "r" (negmask.v32), "m" (*mem32)); \ 174 return (get_##N(&old, mem)); \ 175 } 176 177 EMIT_LOCK_TEST_AND_SET_N(1, uint8_t) 178 EMIT_LOCK_TEST_AND_SET_N(2, uint16_t) 179 180 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \ 181 uintN_t \ 182 __sync_val_compare_and_swap_##N(uintN_t *mem, uintN_t expected, \ 183 uintN_t desired) \ 184 { \ 185 uint32_t *mem32; \ 186 reg_t expected32, desired32, posmask, old; \ 187 uint32_t negmask, temp; \ 188 \ 189 mem32 = round_to_word(mem); \ 190 expected32.v32 = 0x00000000; \ 191 put_##N(&expected32, mem, expected); \ 192 desired32.v32 = 0x00000000; \ 193 put_##N(&desired32, mem, desired); \ 194 posmask.v32 = 0x00000000; \ 195 put_##N(&posmask, mem, ~0); \ 196 negmask = ~posmask.v32; \ 197 \ 198 do_sync(); \ 199 __asm volatile ( \ 200 "1:" \ 201 "\tll %0, %7\n" /* Load old value. */ \ 202 "\tand %2, %5, %0\n" /* Isolate the old value. */ \ 203 "\tbne %2, %3, 2f\n" /* Compare to expected value. */\ 204 "\tand %2, %6, %0\n" /* Remove the old value. */ \ 205 "\tor %2, %4\n" /* Put in the new value. */ \ 206 "\tsc %2, %1\n" /* Attempt to store. */ \ 207 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 208 "2:" \ 209 : "=&r" (old), "=m" (*mem32), "=&r" (temp) \ 210 : "r" (expected32.v32), "r" (desired32.v32), \ 211 "r" (posmask.v32), "r" (negmask), "m" (*mem32)); \ 212 return (get_##N(&old, mem)); \ 213 } 214 215 EMIT_VAL_COMPARE_AND_SWAP_N(1, uint8_t) 216 EMIT_VAL_COMPARE_AND_SWAP_N(2, uint16_t) 217 218 #define EMIT_ARITHMETIC_FETCH_AND_OP_N(N, uintN_t, name, op) \ 219 uintN_t \ 220 __sync_##name##_##N(uintN_t *mem, uintN_t val) \ 221 { \ 222 uint32_t *mem32; \ 223 reg_t val32, posmask, old; \ 224 uint32_t negmask, temp1, temp2; \ 225 \ 226 mem32 = round_to_word(mem); \ 227 val32.v32 = 0x00000000; \ 228 put_##N(&val32, mem, val); \ 229 posmask.v32 = 0x00000000; \ 230 put_##N(&posmask, mem, ~0); \ 231 negmask = ~posmask.v32; \ 232 \ 233 do_sync(); \ 234 __asm volatile ( \ 235 "1:" \ 236 "\tll %0, %7\n" /* Load old value. */ \ 237 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 238 "\tand %2, %5\n" /* Isolate the new value. */ \ 239 "\tand %3, %6, %0\n" /* Remove the old value. */ \ 240 "\tor %2, %3\n" /* Put in the new value. */ \ 241 "\tsc %2, %1\n" /* Attempt to store. */ \ 242 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 243 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 244 "=&r" (temp2) \ 245 : "r" (val32.v32), "r" (posmask.v32), "r" (negmask), \ 246 "m" (*mem32)); \ 247 return (get_##N(&old, mem)); \ 248 } 249 250 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_add, "addu") 251 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_sub, "subu") 252 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_add, "addu") 253 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_sub, "subu") 254 255 #define EMIT_BITWISE_FETCH_AND_OP_N(N, uintN_t, name, op, idempotence) \ 256 uintN_t \ 257 __sync_##name##_##N(uintN_t *mem, uintN_t val) \ 258 { \ 259 uint32_t *mem32; \ 260 reg_t val32, old; \ 261 uint32_t temp; \ 262 \ 263 mem32 = round_to_word(mem); \ 264 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 265 put_##N(&val32, mem, val); \ 266 \ 267 do_sync(); \ 268 __asm volatile ( \ 269 "1:" \ 270 "\tll %0, %4\n" /* Load old value. */ \ 271 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 272 "\tsc %2, %1\n" /* Attempt to store. */ \ 273 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 274 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 275 : "r" (val32.v32), "m" (*mem32)); \ 276 return (get_##N(&old, mem)); \ 277 } 278 279 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_and, "and", 1) 280 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_or, "or", 0) 281 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_xor, "xor", 0) 282 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_and, "and", 1) 283 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_or, "or", 0) 284 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_xor, "xor", 0) 285 286 /* 287 * 32-bit routines. 288 */ 289 290 uint32_t 291 __sync_val_compare_and_swap_4(uint32_t *mem, uint32_t expected, 292 uint32_t desired) 293 { 294 uint32_t old, temp; 295 296 do_sync(); 297 __asm volatile ( 298 "1:" 299 "\tll %0, %5\n" /* Load old value. */ 300 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 301 "\tmove %2, %4\n" /* Value to store. */ 302 "\tsc %2, %1\n" /* Attempt to store. */ 303 "\tbeqz %2, 1b\n" /* Spin if failed. */ 304 "2:" 305 : "=&r" (old), "=m" (*mem), "=&r" (temp) 306 : "r" (expected), "r" (desired), "m" (*mem)); 307 return (old); 308 } 309 310 #define EMIT_FETCH_AND_OP_4(name, op) \ 311 uint32_t \ 312 __sync_##name##_4(uint32_t *mem, uint32_t val) \ 313 { \ 314 uint32_t old, temp; \ 315 \ 316 do_sync(); \ 317 __asm volatile ( \ 318 "1:" \ 319 "\tll %0, %4\n" /* Load old value. */ \ 320 "\t"op"\n" /* Calculate new value. */ \ 321 "\tsc %2, %1\n" /* Attempt to store. */ \ 322 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 323 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 324 : "r" (val), "m" (*mem)); \ 325 return (old); \ 326 } 327 328 EMIT_FETCH_AND_OP_4(lock_test_and_set, "move %2, %3") 329 EMIT_FETCH_AND_OP_4(fetch_and_add, "addu %2, %0, %3") 330 EMIT_FETCH_AND_OP_4(fetch_and_and, "and %2, %0, %3") 331 EMIT_FETCH_AND_OP_4(fetch_and_or, "or %2, %0, %3") 332 EMIT_FETCH_AND_OP_4(fetch_and_sub, "subu %2, %0, %3") 333 EMIT_FETCH_AND_OP_4(fetch_and_xor, "xor %2, %0, %3") 334 335 /* 336 * 64-bit routines. 337 * 338 * Note: All the 64-bit atomic operations are only atomic when running 339 * in 64-bit mode. It is assumed that code compiled for n32 and n64 fits 340 * into this definition and no further safeties are needed. 341 */ 342 343 #if defined(__mips_n32) || defined(__mips_n64) 344 345 uint64_t 346 __sync_val_compare_and_swap_8(uint64_t *mem, uint64_t expected, 347 uint64_t desired) 348 { 349 uint64_t old, temp; 350 351 do_sync(); 352 __asm volatile ( 353 "1:" 354 "\tlld %0, %5\n" /* Load old value. */ 355 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 356 "\tmove %2, %4\n" /* Value to store. */ 357 "\tscd %2, %1\n" /* Attempt to store. */ 358 "\tbeqz %2, 1b\n" /* Spin if failed. */ 359 "2:" 360 : "=&r" (old), "=m" (*mem), "=&r" (temp) 361 : "r" (expected), "r" (desired), "m" (*mem)); 362 return (old); 363 } 364 365 #define EMIT_FETCH_AND_OP_8(name, op) \ 366 uint64_t \ 367 __sync_##name##_8(uint64_t *mem, uint64_t val) \ 368 { \ 369 uint64_t old, temp; \ 370 \ 371 do_sync(); \ 372 __asm volatile ( \ 373 "1:" \ 374 "\tlld %0, %4\n" /* Load old value. */ \ 375 "\t"op"\n" /* Calculate new value. */ \ 376 "\tscd %2, %1\n" /* Attempt to store. */ \ 377 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 378 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 379 : "r" (val), "m" (*mem)); \ 380 return (old); \ 381 } 382 383 EMIT_FETCH_AND_OP_8(lock_test_and_set, "move %2, %3") 384 EMIT_FETCH_AND_OP_8(fetch_and_add, "daddu %2, %0, %3") 385 EMIT_FETCH_AND_OP_8(fetch_and_and, "and %2, %0, %3") 386 EMIT_FETCH_AND_OP_8(fetch_and_or, "or %2, %0, %3") 387 EMIT_FETCH_AND_OP_8(fetch_and_sub, "dsubu %2, %0, %3") 388 EMIT_FETCH_AND_OP_8(fetch_and_xor, "xor %2, %0, %3") 389 390 #endif /* __mips_n32 || __mips_n64 */ 391 392 #endif /* __SYNC_ATOMICS */
Cache object: 9892094b9140ea1e8b5c72e81066a498
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]