|
131
|
1 /* Copyright (C) 2012-2018 Free Software Foundation, Inc.
|
|
111
|
2 Contributed by Richard Henderson <rth@redhat.com>.
|
|
|
3
|
|
|
4 This file is part of the GNU Atomic Library (libatomic).
|
|
|
5
|
|
|
6 Libatomic is free software; you can redistribute it and/or modify it
|
|
|
7 under the terms of the GNU General Public License as published by
|
|
|
8 the Free Software Foundation; either version 3, or (at your option)
|
|
|
9 any later version.
|
|
|
10
|
|
|
11 Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
|
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
14 more details.
|
|
|
15
|
|
|
16 Under Section 7 of GPL version 3, you are granted additional
|
|
|
17 permissions described in the GCC Runtime Library Exception, version
|
|
|
18 3.1, as published by the Free Software Foundation.
|
|
|
19
|
|
|
20 You should have received a copy of the GNU General Public License and
|
|
|
21 a copy of the GCC Runtime Library Exception along with this program;
|
|
|
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
23 <http://www.gnu.org/licenses/>. */
|
|
|
24
|
|
|
25 /* This file contains data types and function declarations that are
|
|
|
26 private to the implementation of libatomic. */
|
|
|
27
|
|
|
28 #ifndef LIBATOMIC_H
|
|
|
29 #define LIBATOMIC_H 1
|
|
|
30
|
|
|
31 #include "auto-config.h"
|
|
|
32 #include <stdbool.h>
|
|
|
33 #include <stdint.h>
|
|
|
34 #include <stddef.h>
|
|
|
35 #include <limits.h>
|
|
|
36 #include <string.h>
|
|
|
37
|
|
|
38
|
|
|
39 /* Symbol concatenation macros. */
|
|
|
40 #define C2_(X,Y) X ## Y
|
|
|
41 #define C2(X,Y) C2_(X,Y)
|
|
|
42 #define C3_(X,Y,Z) X ## Y ## Z
|
|
|
43 #define C3(X,Y,Z) C3_(X,Y,Z)
|
|
|
44 #define C4_(W,X,Y,Z) W ## X ## Y ## Z
|
|
|
45 #define C4(W,X,Y,Z) C4_(W,X,Y,Z)
|
|
|
46
|
|
|
47 /* Stringification macros. */
|
|
|
48 #define S2(X) #X
|
|
|
49 #define S(X) S2(X)
|
|
|
50
|
|
|
51 /* All of the primitive types on which we operate. */
|
|
|
52 typedef unsigned U_1 __attribute__((mode(QI)));
|
|
|
53 #if HAVE_INT2
|
|
|
54 typedef unsigned U_2 __attribute__((mode(HI)));
|
|
|
55 #endif
|
|
|
56 #if HAVE_INT4
|
|
|
57 typedef unsigned U_4 __attribute__((mode(SI)));
|
|
|
58 #endif
|
|
|
59 #if HAVE_INT8
|
|
|
60 typedef unsigned U_8 __attribute__((mode(DI)));
|
|
|
61 #endif
|
|
|
62 #if HAVE_INT16
|
|
|
63 typedef unsigned U_16 __attribute__((mode(TI)));
|
|
|
64 #endif
|
|
|
65
|
|
|
66 /* The widest type that we support. */
|
|
|
67 #if HAVE_INT16
|
|
|
68 # define MAX_SIZE 16
|
|
|
69 #elif HAVE_INT8
|
|
|
70 # define MAX_SIZE 8
|
|
|
71 #elif HAVE_INT4
|
|
|
72 # define MAX_SIZE 4
|
|
|
73 #elif HAVE_INT2
|
|
|
74 # define MAX_SIZE 2
|
|
|
75 #else
|
|
|
76 # define MAX_SIZE 1
|
|
|
77 #endif
|
|
|
78 typedef C2(U_,MAX_SIZE) U_MAX;
|
|
|
79
|
|
|
80 /* Provide dummy fallback types so that stuff is syntactically correct
|
|
|
81 without having to overdo the ifdefs. The code using these should
|
|
|
82 always be protected with the HAVE_INT{n} macros. */
|
|
|
83 #if !HAVE_INT2
|
|
|
84 typedef U_MAX U_2;
|
|
|
85 #endif
|
|
|
86 #if !HAVE_INT4
|
|
|
87 typedef U_MAX U_4;
|
|
|
88 #endif
|
|
|
89 #if !HAVE_INT8
|
|
|
90 typedef U_MAX U_8;
|
|
|
91 #endif
|
|
|
92 #if !HAVE_INT16
|
|
|
93 typedef U_MAX U_16;
|
|
|
94 #endif
|
|
|
95
|
|
|
96 union max_size_u
|
|
|
97 {
|
|
|
98 U_1 b[MAX_SIZE];
|
|
|
99 U_2 i2;
|
|
|
100 U_4 i4;
|
|
|
101 U_8 i8;
|
|
|
102 U_16 i16;
|
|
|
103 };
|
|
|
104
|
|
|
105 /* The "word" size of the machine. */
|
|
|
106 typedef unsigned UWORD __attribute__((mode(word)));
|
|
|
107
|
|
|
108 /* Macros for handing sub-word sized quantities. */
|
|
|
109 #define MASK_1 ((UWORD)0xff)
|
|
|
110 #define MASK_2 ((UWORD)0xffff)
|
|
|
111 #define MASK_4 ((UWORD)0xffffffff)
|
|
|
112 #define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
|
|
|
113 #define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
|
|
|
114 #define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
|
|
|
115
|
|
|
116 /* Most of the files in this library are compiled multiple times with
|
|
|
117 N defined to be a power of 2 between 1 and 16. The SIZE macro is
|
|
|
118 then used to append _N to the symbol being manipulated. */
|
|
|
119 #define SIZE(X) C3(X,_,N)
|
|
|
120 #define WSIZE(X) C3(X,_,WORDSIZE)
|
|
|
121 #define PTR(N,X) ((C2(U_,N) *)X)
|
|
|
122
|
|
|
123 /* And thus, the type on which this compilation will be operating. */
|
|
|
124 #define ITYPE SIZE(I)
|
|
|
125 #define UTYPE SIZE(U)
|
|
|
126
|
|
|
127 /* Utility macros for GCC attributes. */
|
|
|
128 #define UNUSED __attribute__((unused))
|
|
|
129 #ifdef HAVE_ATTRIBUTE_VISIBILITY
|
|
|
130 # define HIDDEN __attribute__((visibility("hidden")))
|
|
|
131 #else
|
|
|
132 # define HIDDEN
|
|
|
133 #endif
|
|
|
134
|
|
|
135 /* Occasionally we have to play games with internal and external symbol
|
|
|
136 names, in order to work around builtin functions of the same name.
|
|
|
137 This macro sets the external name of the function appropriately. */
|
|
|
138 #define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
|
|
|
139
|
|
|
140 /* Locking for a "small" operation. In the bare-metal single processor
|
|
|
141 cases this could be implemented by disabling interrupts. Thus the extra
|
|
|
142 word passed between the two functions, saving the interrupt level.
|
|
|
143 It is assumed that the object being locked does not cross the locking
|
|
|
144 granularity.
|
|
|
145
|
|
|
146 Not actually declared here so that they can be defined static inline
|
|
|
147 in a target-specfic <host-config.h>.
|
|
|
148
|
|
|
149 UWORD protect_start (void *ptr);
|
|
|
150 void protect_end (void *ptr, UWORD);
|
|
|
151 */
|
|
|
152
|
|
|
153 /* Locking for a "large' operation. This should always be some sort of
|
|
|
154 test-and-set operation, as we assume that the interrupt latency would
|
|
|
155 be unreasonably large. */
|
|
|
156 void libat_lock_n (void *ptr, size_t n);
|
|
|
157 void libat_unlock_n (void *ptr, size_t n);
|
|
|
158
|
|
|
159 /* We'll need to declare all of the sized functions a few times... */
|
|
|
160 #define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N))
|
|
|
161 #define DECLARE_ALL_SIZED_(N,T) \
|
|
|
162 DECLARE_1(T, C2(load_,N), (T *mptr, int)); \
|
|
|
163 DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \
|
|
|
164 DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \
|
|
|
165 DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
|
|
|
166 DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \
|
|
|
167 DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \
|
|
|
168 DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \
|
|
|
169 DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \
|
|
|
170 DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \
|
|
|
171 DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \
|
|
|
172 DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \
|
|
|
173 DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \
|
|
|
174 DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \
|
|
|
175 DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \
|
|
|
176 DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \
|
|
|
177 DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \
|
|
|
178 DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int))
|
|
|
179
|
|
|
180 /* All sized operations are implemented in hidden functions prefixed with
|
|
|
181 "libat_". These are either renamed or aliased to the expected prefix
|
|
|
182 of "__atomic". Some amount of renaming is required to avoid hiding or
|
|
|
183 conflicting with the builtins of the same name, but this additional
|
|
|
184 use of hidden symbols (where appropriate) avoids unnecessary PLT entries
|
|
|
185 on relevant targets. */
|
|
|
186
|
|
|
187 #if IFUNC_ALT
|
|
|
188 # define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
|
|
|
189 #elif defined(HAVE_ATTRIBUTE_ALIAS)
|
|
|
190 # define MAN(X) HIDDEN
|
|
|
191 #else
|
|
|
192 # define MAN(X) ASMNAME(C2(__atomic_,X))
|
|
|
193 #endif
|
|
|
194
|
|
|
195 #if !defined(N) && HAVE_IFUNC
|
|
|
196 # define DECLARE_1(RET,NAME,ARGS) \
|
|
|
197 RET C2(libat_,NAME) ARGS MAN(NAME); \
|
|
|
198 RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
|
|
|
199 #else
|
|
|
200 # define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME)
|
|
|
201 #endif
|
|
|
202
|
|
|
203 /* Prefix to use when calling internal, possibly ifunc'ed functions. */
|
|
|
204 #if HAVE_IFUNC
|
|
|
205 # define local_ ifunc_
|
|
|
206 #else
|
|
|
207 # define local_ libat_
|
|
|
208 #endif
|
|
|
209
|
|
|
210 DECLARE_ALL_SIZED(1);
|
|
|
211 DECLARE_ALL_SIZED(2);
|
|
|
212 DECLARE_ALL_SIZED(4);
|
|
|
213 DECLARE_ALL_SIZED(8);
|
|
|
214 DECLARE_ALL_SIZED(16);
|
|
|
215
|
|
|
216 #undef DECLARE_1
|
|
|
217 #undef DECLARE_ALL_SIZED
|
|
|
218 #undef DECLARE_ALL_SIZED_
|
|
|
219
|
|
|
220 /* And the generic sized versions. */
|
|
|
221 void libat_load (size_t, void *, void *, int) MAN(load);
|
|
|
222 void libat_store (size_t, void *, void *, int) MAN(store);
|
|
|
223 void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
|
|
|
224 bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
|
|
|
225 MAN(compare_exchange);
|
|
|
226 bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
|
|
|
227
|
|
|
228 #undef MAN
|
|
|
229
|
|
|
230 #include <host-config.h>
|
|
|
231
|
|
|
232 /* We don't have IFUNC_NCOND until after host-config.h. */
|
|
|
233 #if !HAVE_IFUNC
|
|
|
234 # define IFUNC_NCOND(N) 0
|
|
|
235 #endif
|
|
|
236
|
|
|
237 #if IFUNC_ALT
|
|
|
238 # define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */
|
|
|
239 #elif defined(N) && IFUNC_NCOND(N)
|
|
|
240 # if IFUNC_NCOND(N) == 1
|
|
|
241 # define GEN_SELECTOR(X) \
|
|
|
242 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
|
|
131
|
243 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
|
|
111
|
244 { \
|
|
|
245 if (IFUNC_COND_1) \
|
|
|
246 return C3(libat_,X,_i1); \
|
|
|
247 return C2(libat_,X); \
|
|
|
248 }
|
|
|
249 # elif IFUNC_NCOND(N) == 2
|
|
|
250 # define GEN_SELECTOR(X) \
|
|
|
251 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
|
|
|
252 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
|
|
131
|
253 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
|
|
111
|
254 { \
|
|
|
255 if (IFUNC_COND_1) \
|
|
|
256 return C3(libat_,X,_i1); \
|
|
|
257 if (IFUNC_COND_2) \
|
|
|
258 return C3(libat_,X,_i2); \
|
|
|
259 return C2(libat_,X); \
|
|
|
260 }
|
|
|
261 # elif IFUNC_NCOND(N) == 3
|
|
|
262 # define GEN_SELECTOR(X) \
|
|
|
263 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
|
|
|
264 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
|
|
|
265 extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \
|
|
131
|
266 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
|
|
111
|
267 { \
|
|
|
268 if (IFUNC_COND_1) \
|
|
|
269 return C3(libat_,X,_i1); \
|
|
|
270 if (IFUNC_COND_2) \
|
|
|
271 return C3(libat_,X,_i2); \
|
|
|
272 if (IFUNC_COND_3) \
|
|
|
273 return C3(libat_,X,_i3); \
|
|
|
274 return C2(libat_,X); \
|
|
|
275 }
|
|
|
276 # else
|
|
|
277 # error "Unsupported number of ifunc alternatives."
|
|
|
278 # endif
|
|
|
279 # define EXPORT_ALIAS(X) \
|
|
|
280 GEN_SELECTOR(X) \
|
|
|
281 typeof(C2(libat_,X)) C2(ifunc_,X) \
|
|
|
282 ASMNAME(C2(__atomic_,X)) \
|
|
|
283 __attribute__((ifunc(S(C2(select_,X)))))
|
|
|
284 #elif defined(HAVE_ATTRIBUTE_ALIAS)
|
|
|
285 # define EXPORT_ALIAS(X) \
|
|
|
286 extern typeof(C2(libat_,X)) C2(export_,X) \
|
|
|
287 ASMNAME(C2(__atomic_,X)) \
|
|
|
288 __attribute__((alias(S(C2(libat_,X)))))
|
|
|
289 #else
|
|
|
290 # define EXPORT_ALIAS(X) /* original symbol is exported */
|
|
|
291 #endif
|
|
|
292
|
|
|
293 #endif /* LIBATOMIC_H */
|
