1 /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */
2
3 /*-
4 * Copyright (C) 2003-2004 Olivier Houchard
5 * Copyright (C) 1994-1997 Mark Brinicombe
6 * Copyright (C) 1994 Brini
7 * All rights reserved.
8 *
9 * This code is derived from software written for Brini by Mark Brinicombe
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by Brini.
22 * 4. The name of Brini may not be used to endorse or promote products
23 * derived from this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR
26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
31 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
33 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
34 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * $FreeBSD: releng/11.0/sys/arm/include/atomic-v4.h 300144 2016-05-18 13:09:52Z andrew $
37 */
38
39 #ifndef _MACHINE_ATOMIC_V4_H_
40 #define _MACHINE_ATOMIC_V4_H_
41
42 #ifndef _MACHINE_ATOMIC_H_
43 #error Do not include this file directly, use <machine/atomic.h>
44 #endif
45
46 #if __ARM_ARCH <= 5
47 #define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
48 #define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
49 #define dmb() dsb()
50 #else
51 #error Only use this file with ARMv5 and earlier
52 #endif
53
54 #define mb() dmb()
55 #define wmb() dmb()
56 #define rmb() dmb()
57
58 #define __with_interrupts_disabled(expr) \
59 do { \
60 u_int cpsr_save, tmp; \
61 \
62 __asm __volatile( \
63 "mrs %0, cpsr;" \
64 "orr %1, %0, %2;" \
65 "msr cpsr_fsxc, %1;" \
66 : "=r" (cpsr_save), "=r" (tmp) \
67 : "I" (PSR_I | PSR_F) \
68 : "cc" ); \
69 (expr); \
70 __asm __volatile( \
71 "msr cpsr_fsxc, %0" \
72 : /* no output */ \
73 : "r" (cpsr_save) \
74 : "cc" ); \
75 } while(0)
76
77 static __inline uint32_t
78 __swp(uint32_t val, volatile uint32_t *ptr)
79 {
80 __asm __volatile("swp %0, %2, [%3]"
81 : "=&r" (val), "=m" (*ptr)
82 : "r" (val), "r" (ptr), "m" (*ptr)
83 : "memory");
84 return (val);
85 }
86
87
88 #ifdef _KERNEL
89 #define ARM_HAVE_ATOMIC64
90
91 static __inline void
92 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
93 {
94 __with_interrupts_disabled(*p += val);
95 }
96
97 static __inline void
98 atomic_add_64(volatile u_int64_t *p, u_int64_t val)
99 {
100 __with_interrupts_disabled(*p += val);
101 }
102
103 static __inline void
104 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
105 {
106 __with_interrupts_disabled(*address &= ~clearmask);
107 }
108
109 static __inline void
110 atomic_clear_64(volatile uint64_t *address, uint64_t clearmask)
111 {
112 __with_interrupts_disabled(*address &= ~clearmask);
113 }
114
115 static __inline u_int32_t
116 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
117 {
118 int ret;
119
120 __with_interrupts_disabled(
121 {
122 if (*p == cmpval) {
123 *p = newval;
124 ret = 1;
125 } else {
126 ret = 0;
127 }
128 });
129 return (ret);
130 }
131
132 static __inline u_int64_t
133 atomic_cmpset_64(volatile u_int64_t *p, volatile u_int64_t cmpval, volatile u_int64_t newval)
134 {
135 int ret;
136
137 __with_interrupts_disabled(
138 {
139 if (*p == cmpval) {
140 *p = newval;
141 ret = 1;
142 } else {
143 ret = 0;
144 }
145 });
146 return (ret);
147 }
148
149
150 static __inline uint32_t
151 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
152 {
153 uint32_t value;
154
155 __with_interrupts_disabled(
156 {
157 value = *p;
158 *p += v;
159 });
160 return (value);
161 }
162
163 static __inline uint64_t
164 atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
165 {
166 uint64_t value;
167
168 __with_interrupts_disabled(
169 {
170 value = *p;
171 *p += v;
172 });
173 return (value);
174 }
175
176 static __inline uint64_t
177 atomic_load_64(volatile uint64_t *p)
178 {
179 uint64_t value;
180
181 __with_interrupts_disabled(value = *p);
182 return (value);
183 }
184
185 static __inline void
186 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
187 {
188 __with_interrupts_disabled(*address |= setmask);
189 }
190
191 static __inline void
192 atomic_set_64(volatile uint64_t *address, uint64_t setmask)
193 {
194 __with_interrupts_disabled(*address |= setmask);
195 }
196
197 static __inline void
198 atomic_store_64(volatile uint64_t *p, uint64_t value)
199 {
200 __with_interrupts_disabled(*p = value);
201 }
202
203 static __inline void
204 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
205 {
206 __with_interrupts_disabled(*p -= val);
207 }
208
209 static __inline void
210 atomic_subtract_64(volatile u_int64_t *p, u_int64_t val)
211 {
212 __with_interrupts_disabled(*p -= val);
213 }
214
215 #else /* !_KERNEL */
216
217 static __inline void
218 atomic_add_32(volatile u_int32_t *p, u_int32_t val)
219 {
220 int start, ras_start = ARM_RAS_START;
221
222 __asm __volatile("1:\n"
223 "adr %1, 1b\n"
224 "str %1, [%0]\n"
225 "adr %1, 2f\n"
226 "str %1, [%0, #4]\n"
227 "ldr %1, [%2]\n"
228 "add %1, %1, %3\n"
229 "str %1, [%2]\n"
230 "2:\n"
231 "mov %1, #0\n"
232 "str %1, [%0]\n"
233 "mov %1, #0xffffffff\n"
234 "str %1, [%0, #4]\n"
235 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
236 : : "memory");
237 }
238
239 static __inline void
240 atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
241 {
242 int start, ras_start = ARM_RAS_START;
243
244 __asm __volatile("1:\n"
245 "adr %1, 1b\n"
246 "str %1, [%0]\n"
247 "adr %1, 2f\n"
248 "str %1, [%0, #4]\n"
249 "ldr %1, [%2]\n"
250 "bic %1, %1, %3\n"
251 "str %1, [%2]\n"
252 "2:\n"
253 "mov %1, #0\n"
254 "str %1, [%0]\n"
255 "mov %1, #0xffffffff\n"
256 "str %1, [%0, #4]\n"
257 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
258 : : "memory");
259
260 }
261
262 static __inline u_int32_t
263 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
264 {
265 register int done, ras_start = ARM_RAS_START;
266
267 __asm __volatile("1:\n"
268 "adr %1, 1b\n"
269 "str %1, [%0]\n"
270 "adr %1, 2f\n"
271 "str %1, [%0, #4]\n"
272 "ldr %1, [%2]\n"
273 "cmp %1, %3\n"
274 "streq %4, [%2]\n"
275 "2:\n"
276 "mov %1, #0\n"
277 "str %1, [%0]\n"
278 "mov %1, #0xffffffff\n"
279 "str %1, [%0, #4]\n"
280 "moveq %1, #1\n"
281 "movne %1, #0\n"
282 : "+r" (ras_start), "=r" (done)
283 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory");
284 return (done);
285 }
286
287 static __inline uint32_t
288 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
289 {
290 uint32_t start, tmp, ras_start = ARM_RAS_START;
291
292 __asm __volatile("1:\n"
293 "adr %1, 1b\n"
294 "str %1, [%0]\n"
295 "adr %1, 2f\n"
296 "str %1, [%0, #4]\n"
297 "ldr %1, [%3]\n"
298 "mov %2, %1\n"
299 "add %2, %2, %4\n"
300 "str %2, [%3]\n"
301 "2:\n"
302 "mov %2, #0\n"
303 "str %2, [%0]\n"
304 "mov %2, #0xffffffff\n"
305 "str %2, [%0, #4]\n"
306 : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v)
307 : : "memory");
308 return (start);
309 }
310
311 static __inline void
312 atomic_set_32(volatile uint32_t *address, uint32_t setmask)
313 {
314 int start, ras_start = ARM_RAS_START;
315
316 __asm __volatile("1:\n"
317 "adr %1, 1b\n"
318 "str %1, [%0]\n"
319 "adr %1, 2f\n"
320 "str %1, [%0, #4]\n"
321 "ldr %1, [%2]\n"
322 "orr %1, %1, %3\n"
323 "str %1, [%2]\n"
324 "2:\n"
325 "mov %1, #0\n"
326 "str %1, [%0]\n"
327 "mov %1, #0xffffffff\n"
328 "str %1, [%0, #4]\n"
329
330 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
331 : : "memory");
332 }
333
334 static __inline void
335 atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
336 {
337 int start, ras_start = ARM_RAS_START;
338
339 __asm __volatile("1:\n"
340 "adr %1, 1b\n"
341 "str %1, [%0]\n"
342 "adr %1, 2f\n"
343 "str %1, [%0, #4]\n"
344 "ldr %1, [%2]\n"
345 "sub %1, %1, %3\n"
346 "str %1, [%2]\n"
347 "2:\n"
348 "mov %1, #0\n"
349 "str %1, [%0]\n"
350 "mov %1, #0xffffffff\n"
351 "str %1, [%0, #4]\n"
352
353 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
354 : : "memory");
355 }
356
357 #endif /* _KERNEL */
358
359 static __inline uint32_t
360 atomic_readandclear_32(volatile u_int32_t *p)
361 {
362
363 return (__swp(0, p));
364 }
365
366 static __inline uint32_t
367 atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
368 {
369
370 return (__swp(v, p));
371 }
372
373 #define atomic_cmpset_rel_32 atomic_cmpset_32
374 #define atomic_cmpset_acq_32 atomic_cmpset_32
375 #define atomic_cmpset_rel_64 atomic_cmpset_64
376 #define atomic_cmpset_acq_64 atomic_cmpset_64
377 #define atomic_set_rel_32 atomic_set_32
378 #define atomic_set_acq_32 atomic_set_32
379 #define atomic_clear_rel_32 atomic_clear_32
380 #define atomic_clear_acq_32 atomic_clear_32
381 #define atomic_add_rel_32 atomic_add_32
382 #define atomic_add_acq_32 atomic_add_32
383 #define atomic_subtract_rel_32 atomic_subtract_32
384 #define atomic_subtract_acq_32 atomic_subtract_32
385 #define atomic_store_rel_32 atomic_store_32
386 #define atomic_store_rel_long atomic_store_long
387 #define atomic_load_acq_32 atomic_load_32
388 #define atomic_load_acq_long atomic_load_long
389 #define atomic_add_acq_long atomic_add_long
390 #define atomic_add_rel_long atomic_add_long
391 #define atomic_subtract_acq_long atomic_subtract_long
392 #define atomic_subtract_rel_long atomic_subtract_long
393 #define atomic_clear_acq_long atomic_clear_long
394 #define atomic_clear_rel_long atomic_clear_long
395 #define atomic_set_acq_long atomic_set_long
396 #define atomic_set_rel_long atomic_set_long
397 #define atomic_cmpset_acq_long atomic_cmpset_long
398 #define atomic_cmpset_rel_long atomic_cmpset_long
399 #define atomic_load_acq_long atomic_load_long
400 #undef __with_interrupts_disabled
401
402 static __inline void
403 atomic_add_long(volatile u_long *p, u_long v)
404 {
405
406 atomic_add_32((volatile uint32_t *)p, v);
407 }
408
409 static __inline void
410 atomic_clear_long(volatile u_long *p, u_long v)
411 {
412
413 atomic_clear_32((volatile uint32_t *)p, v);
414 }
415
416 static __inline int
417 atomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe)
418 {
419
420 return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe));
421 }
422
423 static __inline u_long
424 atomic_fetchadd_long(volatile u_long *p, u_long v)
425 {
426
427 return (atomic_fetchadd_32((volatile uint32_t *)p, v));
428 }
429
430 static __inline void
431 atomic_readandclear_long(volatile u_long *p)
432 {
433
434 atomic_readandclear_32((volatile uint32_t *)p);
435 }
436
437 static __inline void
438 atomic_set_long(volatile u_long *p, u_long v)
439 {
440
441 atomic_set_32((volatile uint32_t *)p, v);
442 }
443
444 static __inline void
445 atomic_subtract_long(volatile u_long *p, u_long v)
446 {
447
448 atomic_subtract_32((volatile uint32_t *)p, v);
449 }
450
451 /*
452 * ARMv5 does not support SMP. For both kernel and user modes, only a
453 * compiler barrier is needed for fences, since CPU is always
454 * self-consistent.
455 */
456 static __inline void
457 atomic_thread_fence_acq(void)
458 {
459
460 __compiler_membar();
461 }
462
463 static __inline void
464 atomic_thread_fence_rel(void)
465 {
466
467 __compiler_membar();
468 }
469
470 static __inline void
471 atomic_thread_fence_acq_rel(void)
472 {
473
474 __compiler_membar();
475 }
476
477 static __inline void
478 atomic_thread_fence_seq_cst(void)
479 {
480
481 __compiler_membar();
482 }
483
484 #endif /* _MACHINE_ATOMIC_H_ */
Cache object: 2b3225b95fd1cc179133565708bfb76b
|