1 /*-
2 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Copyright (c) 1998 Doug Rabson
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD: releng/11.1/sys/mips/mips/stdatomic.c 274668 2014-11-18 17:06:56Z imp $");
32
33 #include <sys/stdatomic.h>
34 #include <sys/types.h>
35
36 #if defined(__SYNC_ATOMICS)
37
38 /*
39 * Memory barriers.
40 *
41 * It turns out __sync_synchronize() does not emit any code when used
42 * with GCC 4.2. Implement our own version that does work reliably.
43 *
44 * Although __sync_lock_test_and_set() should only perform an acquire
45 * barrier, make it do a full barrier like the other functions. This
46 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably.
47 */
48
49 static inline void
50 do_sync(void)
51 {
52
53 __asm volatile (
54 #if !defined(_KERNEL) || defined(SMP)
55 ".set noreorder\n"
56 "\tsync\n"
57 "\tnop\n"
58 "\tnop\n"
59 "\tnop\n"
60 "\tnop\n"
61 "\tnop\n"
62 "\tnop\n"
63 "\tnop\n"
64 "\tnop\n"
65 ".set reorder\n"
66 #else /* _KERNEL && !SMP */
67 ""
68 #endif /* !KERNEL || SMP */
69 : : : "memory");
70 }
71
72 typedef union {
73 uint8_t v8[4];
74 uint32_t v32;
75 } reg_t;
76
77 /*
78 * Given a memory address pointing to an 8-bit or 16-bit integer, return
79 * the address of the 32-bit word containing it.
80 */
81
82 static inline uint32_t *
83 round_to_word(void *ptr)
84 {
85
86 return ((uint32_t *)((intptr_t)ptr & ~3));
87 }
88
89 /*
90 * Utility functions for loading and storing 8-bit and 16-bit integers
91 * in 32-bit words at an offset corresponding with the location of the
92 * atomic variable.
93 */
94
95 static inline void
96 put_1(reg_t *r, const uint8_t *offset_ptr, uint8_t val)
97 {
98 size_t offset;
99
100 offset = (intptr_t)offset_ptr & 3;
101 r->v8[offset] = val;
102 }
103
104 static inline uint8_t
105 get_1(const reg_t *r, const uint8_t *offset_ptr)
106 {
107 size_t offset;
108
109 offset = (intptr_t)offset_ptr & 3;
110 return (r->v8[offset]);
111 }
112
113 static inline void
114 put_2(reg_t *r, const uint16_t *offset_ptr, uint16_t val)
115 {
116 size_t offset;
117 union {
118 uint16_t in;
119 uint8_t out[2];
120 } bytes;
121
122 offset = (intptr_t)offset_ptr & 3;
123 bytes.in = val;
124 r->v8[offset] = bytes.out[0];
125 r->v8[offset + 1] = bytes.out[1];
126 }
127
128 static inline uint16_t
129 get_2(const reg_t *r, const uint16_t *offset_ptr)
130 {
131 size_t offset;
132 union {
133 uint8_t in[2];
134 uint16_t out;
135 } bytes;
136
137 offset = (intptr_t)offset_ptr & 3;
138 bytes.in[0] = r->v8[offset];
139 bytes.in[1] = r->v8[offset + 1];
140 return (bytes.out);
141 }
142
143 /*
144 * 8-bit and 16-bit routines.
145 *
146 * These operations are not natively supported by the CPU, so we use
147 * some shifting and bitmasking on top of the 32-bit instructions.
148 */
149
150 #define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t) \
151 uintN_t \
152 __sync_lock_test_and_set_##N(uintN_t *mem, uintN_t val) \
153 { \
154 uint32_t *mem32; \
155 reg_t val32, negmask, old; \
156 uint32_t temp; \
157 \
158 mem32 = round_to_word(mem); \
159 val32.v32 = 0x00000000; \
160 put_##N(&val32, mem, val); \
161 negmask.v32 = 0xffffffff; \
162 put_##N(&negmask, mem, 0); \
163 \
164 do_sync(); \
165 __asm volatile ( \
166 "1:" \
167 "\tll %0, %5\n" /* Load old value. */ \
168 "\tand %2, %4, %0\n" /* Remove the old value. */ \
169 "\tor %2, %3\n" /* Put in the new value. */ \
170 "\tsc %2, %1\n" /* Attempt to store. */ \
171 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
172 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \
173 : "r" (val32.v32), "r" (negmask.v32), "m" (*mem32)); \
174 return (get_##N(&old, mem)); \
175 }
176
177 EMIT_LOCK_TEST_AND_SET_N(1, uint8_t)
178 EMIT_LOCK_TEST_AND_SET_N(2, uint16_t)
179
180 #define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \
181 uintN_t \
182 __sync_val_compare_and_swap_##N(uintN_t *mem, uintN_t expected, \
183 uintN_t desired) \
184 { \
185 uint32_t *mem32; \
186 reg_t expected32, desired32, posmask, old; \
187 uint32_t negmask, temp; \
188 \
189 mem32 = round_to_word(mem); \
190 expected32.v32 = 0x00000000; \
191 put_##N(&expected32, mem, expected); \
192 desired32.v32 = 0x00000000; \
193 put_##N(&desired32, mem, desired); \
194 posmask.v32 = 0x00000000; \
195 put_##N(&posmask, mem, ~0); \
196 negmask = ~posmask.v32; \
197 \
198 do_sync(); \
199 __asm volatile ( \
200 "1:" \
201 "\tll %0, %7\n" /* Load old value. */ \
202 "\tand %2, %5, %0\n" /* Isolate the old value. */ \
203 "\tbne %2, %3, 2f\n" /* Compare to expected value. */\
204 "\tand %2, %6, %0\n" /* Remove the old value. */ \
205 "\tor %2, %4\n" /* Put in the new value. */ \
206 "\tsc %2, %1\n" /* Attempt to store. */ \
207 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
208 "2:" \
209 : "=&r" (old), "=m" (*mem32), "=&r" (temp) \
210 : "r" (expected32.v32), "r" (desired32.v32), \
211 "r" (posmask.v32), "r" (negmask), "m" (*mem32)); \
212 return (get_##N(&old, mem)); \
213 }
214
215 EMIT_VAL_COMPARE_AND_SWAP_N(1, uint8_t)
216 EMIT_VAL_COMPARE_AND_SWAP_N(2, uint16_t)
217
218 #define EMIT_ARITHMETIC_FETCH_AND_OP_N(N, uintN_t, name, op) \
219 uintN_t \
220 __sync_##name##_##N(uintN_t *mem, uintN_t val) \
221 { \
222 uint32_t *mem32; \
223 reg_t val32, posmask, old; \
224 uint32_t negmask, temp1, temp2; \
225 \
226 mem32 = round_to_word(mem); \
227 val32.v32 = 0x00000000; \
228 put_##N(&val32, mem, val); \
229 posmask.v32 = 0x00000000; \
230 put_##N(&posmask, mem, ~0); \
231 negmask = ~posmask.v32; \
232 \
233 do_sync(); \
234 __asm volatile ( \
235 "1:" \
236 "\tll %0, %7\n" /* Load old value. */ \
237 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \
238 "\tand %2, %5\n" /* Isolate the new value. */ \
239 "\tand %3, %6, %0\n" /* Remove the old value. */ \
240 "\tor %2, %3\n" /* Put in the new value. */ \
241 "\tsc %2, %1\n" /* Attempt to store. */ \
242 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
243 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \
244 "=&r" (temp2) \
245 : "r" (val32.v32), "r" (posmask.v32), "r" (negmask), \
246 "m" (*mem32)); \
247 return (get_##N(&old, mem)); \
248 }
249
250 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_add, "addu")
251 EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_sub, "subu")
252 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_add, "addu")
253 EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_sub, "subu")
254
255 #define EMIT_BITWISE_FETCH_AND_OP_N(N, uintN_t, name, op, idempotence) \
256 uintN_t \
257 __sync_##name##_##N(uintN_t *mem, uintN_t val) \
258 { \
259 uint32_t *mem32; \
260 reg_t val32, old; \
261 uint32_t temp; \
262 \
263 mem32 = round_to_word(mem); \
264 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \
265 put_##N(&val32, mem, val); \
266 \
267 do_sync(); \
268 __asm volatile ( \
269 "1:" \
270 "\tll %0, %4\n" /* Load old value. */ \
271 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \
272 "\tsc %2, %1\n" /* Attempt to store. */ \
273 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
274 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \
275 : "r" (val32.v32), "m" (*mem32)); \
276 return (get_##N(&old, mem)); \
277 }
278
279 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_and, "and", 1)
280 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_or, "or", 0)
281 EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_xor, "xor", 0)
282 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_and, "and", 1)
283 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_or, "or", 0)
284 EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_xor, "xor", 0)
285
286 /*
287 * 32-bit routines.
288 */
289
290 uint32_t
291 __sync_val_compare_and_swap_4(uint32_t *mem, uint32_t expected,
292 uint32_t desired)
293 {
294 uint32_t old, temp;
295
296 do_sync();
297 __asm volatile (
298 "1:"
299 "\tll %0, %5\n" /* Load old value. */
300 "\tbne %0, %3, 2f\n" /* Compare to expected value. */
301 "\tmove %2, %4\n" /* Value to store. */
302 "\tsc %2, %1\n" /* Attempt to store. */
303 "\tbeqz %2, 1b\n" /* Spin if failed. */
304 "2:"
305 : "=&r" (old), "=m" (*mem), "=&r" (temp)
306 : "r" (expected), "r" (desired), "m" (*mem));
307 return (old);
308 }
309
310 #define EMIT_FETCH_AND_OP_4(name, op) \
311 uint32_t \
312 __sync_##name##_4(uint32_t *mem, uint32_t val) \
313 { \
314 uint32_t old, temp; \
315 \
316 do_sync(); \
317 __asm volatile ( \
318 "1:" \
319 "\tll %0, %4\n" /* Load old value. */ \
320 "\t"op"\n" /* Calculate new value. */ \
321 "\tsc %2, %1\n" /* Attempt to store. */ \
322 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
323 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
324 : "r" (val), "m" (*mem)); \
325 return (old); \
326 }
327
328 EMIT_FETCH_AND_OP_4(lock_test_and_set, "move %2, %3")
329 EMIT_FETCH_AND_OP_4(fetch_and_add, "addu %2, %0, %3")
330 EMIT_FETCH_AND_OP_4(fetch_and_and, "and %2, %0, %3")
331 EMIT_FETCH_AND_OP_4(fetch_and_or, "or %2, %0, %3")
332 EMIT_FETCH_AND_OP_4(fetch_and_sub, "subu %2, %0, %3")
333 EMIT_FETCH_AND_OP_4(fetch_and_xor, "xor %2, %0, %3")
334
335 /*
336 * 64-bit routines.
337 *
338 * Note: All the 64-bit atomic operations are only atomic when running
339 * in 64-bit mode. It is assumed that code compiled for n32 and n64 fits
340 * into this definition and no further safeties are needed.
341 */
342
343 #if defined(__mips_n32) || defined(__mips_n64)
344
345 uint64_t
346 __sync_val_compare_and_swap_8(uint64_t *mem, uint64_t expected,
347 uint64_t desired)
348 {
349 uint64_t old, temp;
350
351 do_sync();
352 __asm volatile (
353 "1:"
354 "\tlld %0, %5\n" /* Load old value. */
355 "\tbne %0, %3, 2f\n" /* Compare to expected value. */
356 "\tmove %2, %4\n" /* Value to store. */
357 "\tscd %2, %1\n" /* Attempt to store. */
358 "\tbeqz %2, 1b\n" /* Spin if failed. */
359 "2:"
360 : "=&r" (old), "=m" (*mem), "=&r" (temp)
361 : "r" (expected), "r" (desired), "m" (*mem));
362 return (old);
363 }
364
365 #define EMIT_FETCH_AND_OP_8(name, op) \
366 uint64_t \
367 __sync_##name##_8(uint64_t *mem, uint64_t val) \
368 { \
369 uint64_t old, temp; \
370 \
371 do_sync(); \
372 __asm volatile ( \
373 "1:" \
374 "\tlld %0, %4\n" /* Load old value. */ \
375 "\t"op"\n" /* Calculate new value. */ \
376 "\tscd %2, %1\n" /* Attempt to store. */ \
377 "\tbeqz %2, 1b\n" /* Spin if failed. */ \
378 : "=&r" (old), "=m" (*mem), "=&r" (temp) \
379 : "r" (val), "m" (*mem)); \
380 return (old); \
381 }
382
383 EMIT_FETCH_AND_OP_8(lock_test_and_set, "move %2, %3")
384 EMIT_FETCH_AND_OP_8(fetch_and_add, "daddu %2, %0, %3")
385 EMIT_FETCH_AND_OP_8(fetch_and_and, "and %2, %0, %3")
386 EMIT_FETCH_AND_OP_8(fetch_and_or, "or %2, %0, %3")
387 EMIT_FETCH_AND_OP_8(fetch_and_sub, "dsubu %2, %0, %3")
388 EMIT_FETCH_AND_OP_8(fetch_and_xor, "xor %2, %0, %3")
389
390 #endif /* __mips_n32 || __mips_n64 */
391
392 #endif /* __SYNC_ATOMICS */
Cache object: 9892094b9140ea1e8b5c72e81066a498
|