|
145
|
1 // GNU D Compiler emulated TLS routines.
|
|
|
2 // Copyright (C) 2019-2020 Free Software Foundation, Inc.
|
|
|
3
|
|
|
4 // GCC is free software; you can redistribute it and/or modify it under
|
|
|
5 // the terms of the GNU General Public License as published by the Free
|
|
|
6 // Software Foundation; either version 3, or (at your option) any later
|
|
|
7 // version.
|
|
|
8
|
|
|
9 // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
10 // WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
12 // for more details.
|
|
|
13
|
|
|
14 // Under Section 7 of GPL version 3, you are granted additional
|
|
|
15 // permissions described in the GCC Runtime Library Exception, version
|
|
|
16 // 3.1, as published by the Free Software Foundation.
|
|
|
17
|
|
|
18 // You should have received a copy of the GNU General Public License and
|
|
|
19 // a copy of the GCC Runtime Library Exception along with this program;
|
|
|
20 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
21 // <http://www.gnu.org/licenses/>.
|
|
|
22
|
|
|
23 // This code is based on the libgcc emutls.c emulated TLS support.
|
|
|
24
|
|
|
25 module gcc.emutls;
|
|
|
26
|
|
|
27 import core.atomic, core.stdc.stdlib, core.stdc.string, core.sync.mutex;
|
|
|
28 import rt.util.container.array, rt.util.container.hashtab;
|
|
|
29 import core.internal.traits : classInstanceAlignment;
|
|
|
30 import gcc.builtins, gcc.gthread;
|
|
|
31
|
|
|
32 version (GNU_EMUTLS): private:
|
|
|
33
|
|
|
34 alias word = __builtin_machine_uint;
|
|
|
35 alias pointer = __builtin_pointer_uint;
|
|
|
36 alias TlsArray = Array!(void**);
|
|
|
37
|
|
|
38 /*
|
|
|
39 * TLS control data emitted by GCC for every TLS variable.
|
|
|
40 */
|
|
|
41 struct __emutls_object
|
|
|
42 {
|
|
|
43 word size;
|
|
|
44 word align_;
|
|
|
45 union
|
|
|
46 {
|
|
|
47 pointer offset;
|
|
|
48 void* ptr;
|
|
|
49 }
|
|
|
50
|
|
|
51 ubyte* templ;
|
|
|
52 }
|
|
|
53
|
|
|
54 // Per-thread key to obtain the per-thread TLS variable array
|
|
|
55 __gshared __gthread_key_t emutlsKey;
|
|
|
56 // Largest, currently assigned TLS variable offset
|
|
|
57 __gshared pointer emutlsMaxOffset = 0;
|
|
|
58 // Contains the size of the TLS variables (for GC)
|
|
|
59 __gshared Array!word emutlsSizes;
|
|
|
60 // Contains the TLS variable array for single-threaded apps
|
|
|
61 __gshared TlsArray singleArray;
|
|
|
62 // List of all currently alive TlsArrays (for GC)
|
|
|
63 __gshared HashTab!(TlsArray*, TlsArray*) emutlsArrays;
|
|
|
64
|
|
|
65 // emutlsMutex Mutex + @nogc handling
|
|
|
66 enum mutexAlign = classInstanceAlignment!Mutex;
|
|
|
67 enum mutexClassInstanceSize = __traits(classInstanceSize, Mutex);
|
|
|
68 __gshared align(mutexAlign) void[mutexClassInstanceSize] _emutlsMutex;
|
|
|
69
|
|
|
70 @property Mutex emutlsMutex() nothrow @nogc
|
|
|
71 {
|
|
|
72 return cast(Mutex) _emutlsMutex.ptr;
|
|
|
73 }
|
|
|
74
|
|
|
75 /*
|
|
|
76 * Global (de)initialization functions
|
|
|
77 */
|
|
|
78 extern (C) void _d_emutls_init() nothrow @nogc
|
|
|
79 {
|
|
|
80 memcpy(_emutlsMutex.ptr, typeid(Mutex).initializer.ptr, _emutlsMutex.length);
|
|
|
81 (cast(Mutex) _emutlsMutex.ptr).__ctor();
|
|
|
82
|
|
|
83 if (__gthread_key_create(&emutlsKey, &emutlsDestroyThread) != 0)
|
|
|
84 abort();
|
|
|
85 }
|
|
|
86
|
|
|
87 __gshared __gthread_once_t initOnce = GTHREAD_ONCE_INIT;
|
|
|
88
|
|
|
89 /*
|
|
|
90 * emutls main entrypoint, called by GCC for each TLS variable access.
|
|
|
91 */
|
|
|
92 extern (C) void* __emutls_get_address(shared __emutls_object* obj) nothrow @nogc
|
|
|
93 {
|
|
|
94 pointer offset;
|
|
|
95 if (__gthread_active_p())
|
|
|
96 {
|
|
|
97 // Obtain the offset index into the TLS array (same for all-threads)
|
|
|
98 // for requested var. If it is unset, obtain a new offset index.
|
|
|
99 offset = atomicLoad!(MemoryOrder.acq, pointer)(obj.offset);
|
|
|
100 if (__builtin_expect(offset == 0, 0))
|
|
|
101 {
|
|
|
102 __gthread_once(&initOnce, &_d_emutls_init);
|
|
|
103 emutlsMutex.lock_nothrow();
|
|
|
104
|
|
|
105 offset = obj.offset;
|
|
|
106 if (offset == 0)
|
|
|
107 {
|
|
|
108 offset = ++emutlsMaxOffset;
|
|
|
109
|
|
|
110 emutlsSizes.ensureLength(offset);
|
|
|
111 // Note: it's important that we copy any data from obj and
|
|
|
112 // do not keep an reference to obj itself: If a library is
|
|
|
113 // unloaded, its tls variables are not removed from the arrays
|
|
|
114 // and the GC will still scan these. If we then try to reference
|
|
|
115 // a pointer to the data segment of an unloaded library, this
|
|
|
116 // will crash.
|
|
|
117 emutlsSizes[offset - 1] = obj.size;
|
|
|
118
|
|
|
119 atomicStore!(MemoryOrder.rel, pointer)(obj.offset, offset);
|
|
|
120 }
|
|
|
121 emutlsMutex.unlock_nothrow();
|
|
|
122 }
|
|
|
123 }
|
|
|
124 // For single-threaded systems, don't synchronize
|
|
|
125 else
|
|
|
126 {
|
|
|
127 if (__builtin_expect(obj.offset == 0, 0))
|
|
|
128 {
|
|
|
129 offset = ++emutlsMaxOffset;
|
|
|
130
|
|
|
131 emutlsSizes.ensureLength(offset);
|
|
|
132 emutlsSizes[offset - 1] = obj.size;
|
|
|
133
|
|
|
134 obj.offset = offset;
|
|
|
135 }
|
|
|
136 }
|
|
|
137
|
|
|
138 TlsArray* arr;
|
|
|
139 if (__gthread_active_p())
|
|
|
140 arr = cast(TlsArray*) __gthread_getspecific(emutlsKey);
|
|
|
141 else
|
|
|
142 arr = &singleArray;
|
|
|
143
|
|
|
144 // This will always be false for singleArray
|
|
|
145 if (__builtin_expect(arr == null, 0))
|
|
|
146 {
|
|
|
147 arr = mallocTlsArray(offset);
|
|
|
148 __gthread_setspecific(emutlsKey, arr);
|
|
|
149 emutlsMutex.lock_nothrow();
|
|
|
150 emutlsArrays[arr] = arr;
|
|
|
151 emutlsMutex.unlock_nothrow();
|
|
|
152 }
|
|
|
153 // Check if we have to grow the per-thread array
|
|
|
154 else if (__builtin_expect(offset > arr.length, 0))
|
|
|
155 {
|
|
|
156 (*arr).ensureLength(offset);
|
|
|
157 }
|
|
|
158
|
|
|
159 // Offset 0 is used as a not-initialized marker above. In the
|
|
|
160 // TLS array, we start at 0.
|
|
|
161 auto index = offset - 1;
|
|
|
162
|
|
|
163 // Get the per-thread pointer from the TLS array
|
|
|
164 void** ret = (*arr)[index];
|
|
|
165 if (__builtin_expect(ret == null, 0))
|
|
|
166 {
|
|
|
167 // Initial access, have to allocate the storage
|
|
|
168 ret = emutlsAlloc(obj);
|
|
|
169 (*arr)[index] = ret;
|
|
|
170 }
|
|
|
171
|
|
|
172 return ret;
|
|
|
173 }
|
|
|
174
|
|
|
175 // 1:1 copy from libgcc emutls.c
|
|
|
176 extern (C) void __emutls_register_common(__emutls_object* obj, word size, word align_, ubyte* templ) nothrow @nogc
|
|
|
177 {
|
|
|
178 if (obj.size < size)
|
|
|
179 {
|
|
|
180 obj.size = size;
|
|
|
181 obj.templ = null;
|
|
|
182 }
|
|
|
183 if (obj.align_ < align_)
|
|
|
184 obj.align_ = align_;
|
|
|
185 if (templ && size == obj.size)
|
|
|
186 obj.templ = templ;
|
|
|
187 }
|
|
|
188
|
|
|
189 // 1:1 copy from libgcc emutls.c
|
|
|
190 void** emutlsAlloc(shared __emutls_object* obj) nothrow @nogc
|
|
|
191 {
|
|
|
192 void* ptr;
|
|
|
193 void* ret;
|
|
|
194 enum pointerSize = (void*).sizeof;
|
|
|
195
|
|
|
196 /* We could use here posix_memalign if available and adjust
|
|
|
197 emutls_destroy accordingly. */
|
|
|
198 if ((cast() obj).align_ <= pointerSize)
|
|
|
199 {
|
|
|
200 ptr = malloc((cast() obj).size + pointerSize);
|
|
|
201 if (ptr == null)
|
|
|
202 abort();
|
|
|
203 (cast(void**) ptr)[0] = ptr;
|
|
|
204 ret = ptr + pointerSize;
|
|
|
205 }
|
|
|
206 else
|
|
|
207 {
|
|
|
208 ptr = malloc(obj.size + pointerSize + obj.align_ - 1);
|
|
|
209 if (ptr == null)
|
|
|
210 abort();
|
|
|
211 ret = cast(void*)((cast(pointer)(ptr + pointerSize + obj.align_ - 1)) & ~cast(
|
|
|
212 pointer)(obj.align_ - 1));
|
|
|
213 (cast(void**) ret)[-1] = ptr;
|
|
|
214 }
|
|
|
215
|
|
|
216 if (obj.templ)
|
|
|
217 memcpy(ret, cast(ubyte*) obj.templ, cast() obj.size);
|
|
|
218 else
|
|
|
219 memset(ret, 0, cast() obj.size);
|
|
|
220
|
|
|
221 return cast(void**) ret;
|
|
|
222 }
|
|
|
223
|
|
|
224 /*
|
|
|
225 * When a thread has finished, remove the TLS array from the GC
|
|
|
226 * scan list emutlsArrays, free all allocated TLS variables and
|
|
|
227 * finally free the array.
|
|
|
228 */
|
|
|
229 extern (C) void emutlsDestroyThread(void* ptr) nothrow @nogc
|
|
|
230 {
|
|
|
231 auto arr = cast(TlsArray*) ptr;
|
|
|
232 emutlsMutex.lock_nothrow();
|
|
|
233 emutlsArrays.remove(arr);
|
|
|
234 emutlsMutex.unlock_nothrow();
|
|
|
235
|
|
|
236 foreach (entry; *arr)
|
|
|
237 {
|
|
|
238 if (entry)
|
|
|
239 free(entry[-1]);
|
|
|
240 }
|
|
|
241
|
|
|
242 free(arr);
|
|
|
243 }
|
|
|
244
|
|
|
245 /*
|
|
|
246 * Allocate a new TLS array, set length according to offset.
|
|
|
247 */
|
|
|
248 TlsArray* mallocTlsArray(pointer offset = 0) nothrow @nogc
|
|
|
249 {
|
|
|
250 static assert(TlsArray.alignof == (void*).alignof);
|
|
|
251 void[] data = malloc(TlsArray.sizeof)[0 .. TlsArray.sizeof];
|
|
|
252 if (data.ptr == null)
|
|
|
253 abort();
|
|
|
254
|
|
|
255 static immutable TlsArray init = TlsArray.init;
|
|
|
256 memcpy(data.ptr, &init, data.length);
|
|
|
257 (cast(TlsArray*) data).length = 32;
|
|
|
258 return cast(TlsArray*) data.ptr;
|
|
|
259 }
|
|
|
260
|
|
|
261 /*
|
|
|
262 * Make sure array is large enough to hold an entry for offset.
|
|
|
263 * Note: the array index will be offset - 1!
|
|
|
264 */
|
|
|
265 void ensureLength(Value)(ref Array!(Value) arr, size_t offset) nothrow @nogc
|
|
|
266 {
|
|
|
267 // index is offset-1
|
|
|
268 if (offset > arr.length)
|
|
|
269 {
|
|
|
270 auto newSize = arr.length * 2;
|
|
|
271 if (offset > newSize)
|
|
|
272 newSize = offset + 32;
|
|
|
273 arr.length = newSize;
|
|
|
274 }
|
|
|
275 }
|
|
|
276
|
|
|
277 // Public interface
|
|
|
278 public:
|
|
|
279 void _d_emutls_scan(scope void delegate(void* pbeg, void* pend) nothrow cb) nothrow
|
|
|
280 {
|
|
|
281 void scanArray(scope TlsArray* arr) nothrow
|
|
|
282 {
|
|
|
283 foreach (index, entry; *arr)
|
|
|
284 {
|
|
|
285 auto ptr = cast(void*) entry;
|
|
|
286 if (ptr)
|
|
|
287 cb(ptr, ptr + emutlsSizes[index]);
|
|
|
288 }
|
|
|
289 }
|
|
|
290
|
|
|
291 __gthread_once(&initOnce, &_d_emutls_init);
|
|
|
292 emutlsMutex.lock_nothrow();
|
|
|
293 // this code is effectively nothrow
|
|
|
294 try
|
|
|
295 {
|
|
|
296 foreach (arr, value; emutlsArrays)
|
|
|
297 {
|
|
|
298 scanArray(arr);
|
|
|
299 }
|
|
|
300 }
|
|
|
301 catch (Exception)
|
|
|
302 {
|
|
|
303 }
|
|
|
304 emutlsMutex.unlock_nothrow();
|
|
|
305 scanArray(&singleArray);
|
|
|
306 }
|
|
|
307
|
|
|
308 // Call this after druntime has been unloaded
|
|
|
309 void _d_emutls_destroy() nothrow @nogc
|
|
|
310 {
|
|
|
311 if (__gthread_key_delete(emutlsKey) != 0)
|
|
|
312 abort();
|
|
|
313
|
|
|
314 (cast(Mutex) _emutlsMutex.ptr).__dtor();
|
|
|
315 destroy(emutlsArrays);
|
|
|
316 }
|
