|
0
|
1 /* Linux-specific atomic operations for ARM EABI.
|
|
|
2 Copyright (C) 2008, 2009 Free Software Foundation, Inc.
|
|
|
3 Contributed by CodeSourcery.
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 Under Section 7 of GPL version 3, you are granted additional
|
|
|
18 permissions described in the GCC Runtime Library Exception, version
|
|
|
19 3.1, as published by the Free Software Foundation.
|
|
|
20
|
|
|
21 You should have received a copy of the GNU General Public License and
|
|
|
22 a copy of the GCC Runtime Library Exception along with this program;
|
|
|
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
24 <http://www.gnu.org/licenses/>. */
|
|
|
25
|
|
|
26 /* Kernel helper for compare-and-exchange. */
|
|
|
27 typedef int (__kernel_cmpxchg_t) (int oldval, int newval, int *ptr);
|
|
|
28 #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *) 0xffff0fc0)
|
|
|
29
|
|
|
30 /* Kernel helper for memory barrier. */
|
|
|
31 typedef void (__kernel_dmb_t) (void);
|
|
|
32 #define __kernel_dmb (*(__kernel_dmb_t *) 0xffff0fa0)
|
|
|
33
|
|
|
34 /* Note: we implement byte, short and int versions of atomic operations using
|
|
|
35 the above kernel helpers, but there is no support for "long long" (64-bit)
|
|
|
36 operations as yet. */
|
|
|
37
|
|
|
38 #define HIDDEN __attribute__ ((visibility ("hidden")))
|
|
|
39
|
|
|
40 #ifdef __ARMEL__
|
|
|
41 #define INVERT_MASK_1 0
|
|
|
42 #define INVERT_MASK_2 0
|
|
|
43 #else
|
|
|
44 #define INVERT_MASK_1 24
|
|
|
45 #define INVERT_MASK_2 16
|
|
|
46 #endif
|
|
|
47
|
|
|
48 #define MASK_1 0xffu
|
|
|
49 #define MASK_2 0xffffu
|
|
|
50
|
|
|
51 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
|
|
|
52 int HIDDEN \
|
|
|
53 __sync_fetch_and_##OP##_4 (int *ptr, int val) \
|
|
|
54 { \
|
|
|
55 int failure, tmp; \
|
|
|
56 \
|
|
|
57 do { \
|
|
|
58 tmp = *ptr; \
|
|
|
59 failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr); \
|
|
|
60 } while (failure != 0); \
|
|
|
61 \
|
|
|
62 return tmp; \
|
|
|
63 }
|
|
|
64
|
|
|
65 FETCH_AND_OP_WORD (add, , +)
|
|
|
66 FETCH_AND_OP_WORD (sub, , -)
|
|
|
67 FETCH_AND_OP_WORD (or, , |)
|
|
|
68 FETCH_AND_OP_WORD (and, , &)
|
|
|
69 FETCH_AND_OP_WORD (xor, , ^)
|
|
|
70 FETCH_AND_OP_WORD (nand, ~, &)
|
|
|
71
|
|
|
72 #define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH
|
|
|
73 #define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH
|
|
|
74
|
|
|
75 /* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for
|
|
|
76 subword-sized quantities. */
|
|
|
77
|
|
|
78 #define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN) \
|
|
|
79 TYPE HIDDEN \
|
|
|
80 NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val) \
|
|
|
81 { \
|
|
|
82 int *wordptr = (int *) ((unsigned int) ptr & ~3); \
|
|
|
83 unsigned int mask, shift, oldval, newval; \
|
|
|
84 int failure; \
|
|
|
85 \
|
|
|
86 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
|
|
|
87 mask = MASK_##WIDTH << shift; \
|
|
|
88 \
|
|
|
89 do { \
|
|
|
90 oldval = *wordptr; \
|
|
|
91 newval = ((PFX_OP ((oldval & mask) >> shift) \
|
|
|
92 INF_OP (unsigned int) val) << shift) & mask; \
|
|
|
93 newval |= oldval & ~mask; \
|
|
|
94 failure = __kernel_cmpxchg (oldval, newval, wordptr); \
|
|
|
95 } while (failure != 0); \
|
|
|
96 \
|
|
|
97 return (RETURN & mask) >> shift; \
|
|
|
98 }
|
|
|
99
|
|
|
100 SUBWORD_SYNC_OP (add, , +, short, 2, oldval)
|
|
|
101 SUBWORD_SYNC_OP (sub, , -, short, 2, oldval)
|
|
|
102 SUBWORD_SYNC_OP (or, , |, short, 2, oldval)
|
|
|
103 SUBWORD_SYNC_OP (and, , &, short, 2, oldval)
|
|
|
104 SUBWORD_SYNC_OP (xor, , ^, short, 2, oldval)
|
|
|
105 SUBWORD_SYNC_OP (nand, ~, &, short, 2, oldval)
|
|
|
106
|
|
|
107 SUBWORD_SYNC_OP (add, , +, char, 1, oldval)
|
|
|
108 SUBWORD_SYNC_OP (sub, , -, char, 1, oldval)
|
|
|
109 SUBWORD_SYNC_OP (or, , |, char, 1, oldval)
|
|
|
110 SUBWORD_SYNC_OP (and, , &, char, 1, oldval)
|
|
|
111 SUBWORD_SYNC_OP (xor, , ^, char, 1, oldval)
|
|
|
112 SUBWORD_SYNC_OP (nand, ~, &, char, 1, oldval)
|
|
|
113
|
|
|
114 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
|
|
|
115 int HIDDEN \
|
|
|
116 __sync_##OP##_and_fetch_4 (int *ptr, int val) \
|
|
|
117 { \
|
|
|
118 int tmp, failure; \
|
|
|
119 \
|
|
|
120 do { \
|
|
|
121 tmp = *ptr; \
|
|
|
122 failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr); \
|
|
|
123 } while (failure != 0); \
|
|
|
124 \
|
|
|
125 return PFX_OP tmp INF_OP val; \
|
|
|
126 }
|
|
|
127
|
|
|
128 OP_AND_FETCH_WORD (add, , +)
|
|
|
129 OP_AND_FETCH_WORD (sub, , -)
|
|
|
130 OP_AND_FETCH_WORD (or, , |)
|
|
|
131 OP_AND_FETCH_WORD (and, , &)
|
|
|
132 OP_AND_FETCH_WORD (xor, , ^)
|
|
|
133 OP_AND_FETCH_WORD (nand, ~, &)
|
|
|
134
|
|
|
135 SUBWORD_SYNC_OP (add, , +, short, 2, newval)
|
|
|
136 SUBWORD_SYNC_OP (sub, , -, short, 2, newval)
|
|
|
137 SUBWORD_SYNC_OP (or, , |, short, 2, newval)
|
|
|
138 SUBWORD_SYNC_OP (and, , &, short, 2, newval)
|
|
|
139 SUBWORD_SYNC_OP (xor, , ^, short, 2, newval)
|
|
|
140 SUBWORD_SYNC_OP (nand, ~, &, short, 2, newval)
|
|
|
141
|
|
|
142 SUBWORD_SYNC_OP (add, , +, char, 1, newval)
|
|
|
143 SUBWORD_SYNC_OP (sub, , -, char, 1, newval)
|
|
|
144 SUBWORD_SYNC_OP (or, , |, char, 1, newval)
|
|
|
145 SUBWORD_SYNC_OP (and, , &, char, 1, newval)
|
|
|
146 SUBWORD_SYNC_OP (xor, , ^, char, 1, newval)
|
|
|
147 SUBWORD_SYNC_OP (nand, ~, &, char, 1, newval)
|
|
|
148
|
|
|
149 int HIDDEN
|
|
|
150 __sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval)
|
|
|
151 {
|
|
|
152 int actual_oldval, fail;
|
|
|
153
|
|
|
154 while (1)
|
|
|
155 {
|
|
|
156 actual_oldval = *ptr;
|
|
|
157
|
|
|
158 if (oldval != actual_oldval)
|
|
|
159 return actual_oldval;
|
|
|
160
|
|
|
161 fail = __kernel_cmpxchg (actual_oldval, newval, ptr);
|
|
|
162
|
|
|
163 if (!fail)
|
|
|
164 return oldval;
|
|
|
165 }
|
|
|
166 }
|
|
|
167
|
|
|
168 #define SUBWORD_VAL_CAS(TYPE, WIDTH) \
|
|
|
169 TYPE HIDDEN \
|
|
|
170 __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
|
|
|
171 TYPE newval) \
|
|
|
172 { \
|
|
|
173 int *wordptr = (int *)((unsigned int) ptr & ~3), fail; \
|
|
|
174 unsigned int mask, shift, actual_oldval, actual_newval; \
|
|
|
175 \
|
|
|
176 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
|
|
|
177 mask = MASK_##WIDTH << shift; \
|
|
|
178 \
|
|
|
179 while (1) \
|
|
|
180 { \
|
|
|
181 actual_oldval = *wordptr; \
|
|
|
182 \
|
|
|
183 if (((actual_oldval & mask) >> shift) != (unsigned int) oldval) \
|
|
|
184 return (actual_oldval & mask) >> shift; \
|
|
|
185 \
|
|
|
186 actual_newval = (actual_oldval & ~mask) \
|
|
|
187 | (((unsigned int) newval << shift) & mask); \
|
|
|
188 \
|
|
|
189 fail = __kernel_cmpxchg (actual_oldval, actual_newval, \
|
|
|
190 wordptr); \
|
|
|
191 \
|
|
|
192 if (!fail) \
|
|
|
193 return oldval; \
|
|
|
194 } \
|
|
|
195 }
|
|
|
196
|
|
|
197 SUBWORD_VAL_CAS (short, 2)
|
|
|
198 SUBWORD_VAL_CAS (char, 1)
|
|
|
199
|
|
|
200 typedef unsigned char bool;
|
|
|
201
|
|
|
202 bool HIDDEN
|
|
|
203 __sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval)
|
|
|
204 {
|
|
|
205 int failure = __kernel_cmpxchg (oldval, newval, ptr);
|
|
|
206 return (failure == 0);
|
|
|
207 }
|
|
|
208
|
|
|
209 #define SUBWORD_BOOL_CAS(TYPE, WIDTH) \
|
|
|
210 bool HIDDEN \
|
|
|
211 __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
|
|
|
212 TYPE newval) \
|
|
|
213 { \
|
|
|
214 TYPE actual_oldval \
|
|
|
215 = __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval); \
|
|
|
216 return (oldval == actual_oldval); \
|
|
|
217 }
|
|
|
218
|
|
|
219 SUBWORD_BOOL_CAS (short, 2)
|
|
|
220 SUBWORD_BOOL_CAS (char, 1)
|
|
|
221
|
|
|
222 void HIDDEN
|
|
|
223 __sync_synchronize (void)
|
|
|
224 {
|
|
|
225 __kernel_dmb ();
|
|
|
226 }
|
|
|
227
|
|
|
228 int HIDDEN
|
|
|
229 __sync_lock_test_and_set_4 (int *ptr, int val)
|
|
|
230 {
|
|
|
231 int failure, oldval;
|
|
|
232
|
|
|
233 do {
|
|
|
234 oldval = *ptr;
|
|
|
235 failure = __kernel_cmpxchg (oldval, val, ptr);
|
|
|
236 } while (failure != 0);
|
|
|
237
|
|
|
238 return oldval;
|
|
|
239 }
|
|
|
240
|
|
|
241 #define SUBWORD_TEST_AND_SET(TYPE, WIDTH) \
|
|
|
242 TYPE HIDDEN \
|
|
|
243 __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \
|
|
|
244 { \
|
|
|
245 int failure; \
|
|
|
246 unsigned int oldval, newval, shift, mask; \
|
|
|
247 int *wordptr = (int *) ((unsigned int) ptr & ~3); \
|
|
|
248 \
|
|
|
249 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
|
|
|
250 mask = MASK_##WIDTH << shift; \
|
|
|
251 \
|
|
|
252 do { \
|
|
|
253 oldval = *wordptr; \
|
|
|
254 newval = (oldval & ~mask) \
|
|
|
255 | (((unsigned int) val << shift) & mask); \
|
|
|
256 failure = __kernel_cmpxchg (oldval, newval, wordptr); \
|
|
|
257 } while (failure != 0); \
|
|
|
258 \
|
|
|
259 return (oldval & mask) >> shift; \
|
|
|
260 }
|
|
|
261
|
|
|
262 SUBWORD_TEST_AND_SET (short, 2)
|
|
|
263 SUBWORD_TEST_AND_SET (char, 1)
|
|
|
264
|
|
|
265 #define SYNC_LOCK_RELEASE(TYPE, WIDTH) \
|
|
|
266 void HIDDEN \
|
|
|
267 __sync_lock_release_##WIDTH (TYPE *ptr) \
|
|
|
268 { \
|
|
|
269 *ptr = 0; \
|
|
|
270 __kernel_dmb (); \
|
|
|
271 }
|
|
|
272
|
|
|
273 SYNC_LOCK_RELEASE (int, 4)
|
|
|
274 SYNC_LOCK_RELEASE (short, 2)
|
|
|
275 SYNC_LOCK_RELEASE (char, 1)
|
