|
111
|
1 // Copyright (C) 2002-2017 Free Software Foundation, Inc.
|
|
|
2 //
|
|
|
3 // This file is part of GCC.
|
|
|
4 //
|
|
|
5 // GCC is free software; you can redistribute it and/or modify
|
|
|
6 // it under the terms of the GNU General Public License as published by
|
|
|
7 // the Free Software Foundation; either version 3, or (at your option)
|
|
|
8 // any later version.
|
|
|
9
|
|
|
10 // GCC is distributed in the hope that it will be useful,
|
|
|
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
13 // GNU General Public License for more details.
|
|
|
14
|
|
|
15 // Under Section 7 of GPL version 3, you are granted additional
|
|
|
16 // permissions described in the GCC Runtime Library Exception, version
|
|
|
17 // 3.1, as published by the Free Software Foundation.
|
|
|
18
|
|
|
19 // You should have received a copy of the GNU General Public License and
|
|
|
20 // a copy of the GCC Runtime Library Exception along with this program;
|
|
|
21 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
22 // <http://www.gnu.org/licenses/>.
|
|
|
23
|
|
|
24 // Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
|
|
|
25 // Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
|
|
|
26
|
|
|
27 #include <bits/c++config.h>
|
|
|
28 #include <cxxabi.h>
|
|
|
29 #include <exception>
|
|
|
30 #include <new>
|
|
|
31 #include <ext/atomicity.h>
|
|
|
32 #include <ext/concurrence.h>
|
|
|
33 #include <bits/atomic_lockfree_defines.h>
|
|
|
34 #if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
|
|
|
35 && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
|
|
|
36 # include <climits>
|
|
|
37 # include <syscall.h>
|
|
|
38 # include <unistd.h>
|
|
|
39 # define _GLIBCXX_USE_FUTEX
|
|
|
40 # define _GLIBCXX_FUTEX_WAIT 0
|
|
|
41 # define _GLIBCXX_FUTEX_WAKE 1
|
|
|
42 #endif
|
|
|
43
|
|
|
44 // The IA64/generic ABI uses the first byte of the guard variable.
|
|
|
45 // The ARM EABI uses the least significant bit.
|
|
|
46
|
|
|
47 // Thread-safe static local initialization support.
|
|
|
48 #ifdef __GTHREADS
|
|
|
49 # ifndef _GLIBCXX_USE_FUTEX
|
|
|
50 namespace
|
|
|
51 {
|
|
|
52 // A single mutex controlling all static initializations.
|
|
|
53 static __gnu_cxx::__recursive_mutex* static_mutex;
|
|
|
54
|
|
|
55 typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)]
|
|
|
56 __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex))));
|
|
|
57 fake_recursive_mutex fake_mutex;
|
|
|
58
|
|
|
59 static void init()
|
|
|
60 { static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); }
|
|
|
61
|
|
|
62 __gnu_cxx::__recursive_mutex&
|
|
|
63 get_static_mutex()
|
|
|
64 {
|
|
|
65 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
|
|
|
66 __gthread_once(&once, init);
|
|
|
67 return *static_mutex;
|
|
|
68 }
|
|
|
69
|
|
|
70 // Simple wrapper for exception safety.
|
|
|
71 struct mutex_wrapper
|
|
|
72 {
|
|
|
73 bool unlock;
|
|
|
74 mutex_wrapper() : unlock(true)
|
|
|
75 { get_static_mutex().lock(); }
|
|
|
76
|
|
|
77 ~mutex_wrapper()
|
|
|
78 {
|
|
|
79 if (unlock)
|
|
|
80 static_mutex->unlock();
|
|
|
81 }
|
|
|
82 };
|
|
|
83 }
|
|
|
84 # endif
|
|
|
85
|
|
|
86 # if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
|
|
|
87 namespace
|
|
|
88 {
|
|
|
89 // A single condition variable controlling all static initializations.
|
|
|
90 static __gnu_cxx::__cond* static_cond;
|
|
|
91
|
|
|
92 // using a fake type to avoid initializing a static class.
|
|
|
93 typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)]
|
|
|
94 __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond))));
|
|
|
95 fake_cond_t fake_cond;
|
|
|
96
|
|
|
97 static void init_static_cond()
|
|
|
98 { static_cond = new (&fake_cond) __gnu_cxx::__cond(); }
|
|
|
99
|
|
|
100 __gnu_cxx::__cond&
|
|
|
101 get_static_cond()
|
|
|
102 {
|
|
|
103 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
|
|
|
104 __gthread_once(&once, init_static_cond);
|
|
|
105 return *static_cond;
|
|
|
106 }
|
|
|
107 }
|
|
|
108 # endif
|
|
|
109
|
|
|
110 # ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
|
|
|
111
|
|
|
112 // Test the guard variable with a memory load with
|
|
|
113 // acquire semantics.
|
|
|
114
|
|
|
115 inline bool
|
|
|
116 __test_and_acquire (__cxxabiv1::__guard *g)
|
|
|
117 {
|
|
|
118 unsigned char __c;
|
|
|
119 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
|
|
|
120 __atomic_load (__p, &__c, __ATOMIC_ACQUIRE);
|
|
|
121 (void) __p;
|
|
|
122 return _GLIBCXX_GUARD_TEST(&__c);
|
|
|
123 }
|
|
|
124 # define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
|
|
|
125 # endif
|
|
|
126
|
|
|
127 # ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
|
|
|
128
|
|
|
129 // Set the guard variable to 1 with memory order release semantics.
|
|
|
130
|
|
|
131 inline void
|
|
|
132 __set_and_release (__cxxabiv1::__guard *g)
|
|
|
133 {
|
|
|
134 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
|
|
|
135 unsigned char val = 1;
|
|
|
136 __atomic_store (__p, &val, __ATOMIC_RELEASE);
|
|
|
137 (void) __p;
|
|
|
138 }
|
|
|
139 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
|
|
|
140 # endif
|
|
|
141
|
|
|
142 #else /* !__GTHREADS */
|
|
|
143
|
|
|
144 # undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
|
|
|
145 # undef _GLIBCXX_GUARD_SET_AND_RELEASE
|
|
|
146 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
|
|
|
147
|
|
|
148 #endif /* __GTHREADS */
|
|
|
149
|
|
|
150 //
|
|
|
151 // Here are C++ run-time routines for guarded initialization of static
|
|
|
152 // variables. There are 4 scenarios under which these routines are called:
|
|
|
153 //
|
|
|
154 // 1. Threads not supported (__GTHREADS not defined)
|
|
|
155 // 2. Threads are supported but not enabled at run-time.
|
|
|
156 // 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
|
|
|
157 // 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
|
|
|
158 // primitives we need here.
|
|
|
159 //
|
|
|
160 // The old code supported scenarios 1-3 but was broken since it used a global
|
|
|
161 // mutex for all threads and had the mutex locked during the whole duration of
|
|
|
162 // initialization of a guarded static variable. The following created a
|
|
|
163 // dead-lock with the old code.
|
|
|
164 //
|
|
|
165 // Thread 1 acquires the global mutex.
|
|
|
166 // Thread 1 starts initializing static variable.
|
|
|
167 // Thread 1 creates thread 2 during initialization.
|
|
|
168 // Thread 2 attempts to acquire mutex to initialize another variable.
|
|
|
169 // Thread 2 blocks since thread 1 is locking the mutex.
|
|
|
170 // Thread 1 waits for result from thread 2 and also blocks. A deadlock.
|
|
|
171 //
|
|
|
172 // The new code here can handle this situation and thus is more robust. However,
|
|
|
173 // we need to use the POSIX thread condition variable, which is not supported
|
|
|
174 // in all platforms, notably older versions of Microsoft Windows. The gthr*.h
|
|
|
175 // headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
|
|
|
176 // like condition variables. For platforms that do not support condition
|
|
|
177 // variables, we need to fall back to the old code.
|
|
|
178
|
|
|
179 // If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
|
|
|
180 // only atomic operations are used together with futex syscall.
|
|
|
181 // Valid values of the first integer in guard are:
|
|
|
182 // 0 No thread encountered the guarded init
|
|
|
183 // yet or it has been aborted.
|
|
|
184 // _GLIBCXX_GUARD_BIT The guarded static var has been successfully
|
|
|
185 // initialized.
|
|
|
186 // _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
|
|
|
187 // and no other thread is waiting for its
|
|
|
188 // initialization.
|
|
|
189 // (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
|
|
|
190 // | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
|
|
|
191 // it is initialized.
|
|
|
192
|
|
|
193 namespace __cxxabiv1
|
|
|
194 {
|
|
|
195 #ifdef _GLIBCXX_USE_FUTEX
|
|
|
196 namespace
|
|
|
197 {
|
|
|
198 static inline int __guard_test_bit (const int __byte, const int __val)
|
|
|
199 {
|
|
|
200 union { int __i; char __c[sizeof (int)]; } __u = { 0 };
|
|
|
201 __u.__c[__byte] = __val;
|
|
|
202 return __u.__i;
|
|
|
203 }
|
|
|
204 }
|
|
|
205 #endif
|
|
|
206
|
|
|
207 static inline int
|
|
|
208 init_in_progress_flag(__guard* g)
|
|
|
209 { return ((char *)g)[1]; }
|
|
|
210
|
|
|
211 static inline void
|
|
|
212 set_init_in_progress_flag(__guard* g, int v)
|
|
|
213 { ((char *)g)[1] = v; }
|
|
|
214
|
|
|
215 static inline void
|
|
|
216 throw_recursive_init_exception()
|
|
|
217 {
|
|
|
218 #if __cpp_exceptions
|
|
|
219 throw __gnu_cxx::recursive_init_error();
|
|
|
220 #else
|
|
|
221 // Use __builtin_trap so we don't require abort().
|
|
|
222 __builtin_trap();
|
|
|
223 #endif
|
|
|
224 }
|
|
|
225
|
|
|
226 // acquire() is a helper function used to acquire guard if thread support is
|
|
|
227 // not compiled in or is compiled in but not enabled at run-time.
|
|
|
228 static int
|
|
|
229 acquire(__guard *g)
|
|
|
230 {
|
|
|
231 // Quit if the object is already initialized.
|
|
|
232 if (_GLIBCXX_GUARD_TEST(g))
|
|
|
233 return 0;
|
|
|
234
|
|
|
235 if (init_in_progress_flag(g))
|
|
|
236 throw_recursive_init_exception();
|
|
|
237
|
|
|
238 set_init_in_progress_flag(g, 1);
|
|
|
239 return 1;
|
|
|
240 }
|
|
|
241
|
|
|
242 extern "C"
|
|
|
243 int __cxa_guard_acquire (__guard *g)
|
|
|
244 {
|
|
|
245 #ifdef __GTHREADS
|
|
|
246 // If the target can reorder loads, we need to insert a read memory
|
|
|
247 // barrier so that accesses to the guarded variable happen after the
|
|
|
248 // guard test.
|
|
|
249 if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g))
|
|
|
250 return 0;
|
|
|
251
|
|
|
252 # ifdef _GLIBCXX_USE_FUTEX
|
|
|
253 // If __atomic_* and futex syscall are supported, don't use any global
|
|
|
254 // mutex.
|
|
|
255 if (__gthread_active_p ())
|
|
|
256 {
|
|
|
257 int *gi = (int *) (void *) g;
|
|
|
258 const int guard_bit = _GLIBCXX_GUARD_BIT;
|
|
|
259 const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT;
|
|
|
260 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
|
261
|
|
|
262 while (1)
|
|
|
263 {
|
|
|
264 int expected(0);
|
|
|
265 if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false,
|
|
|
266 __ATOMIC_ACQ_REL,
|
|
|
267 __ATOMIC_ACQUIRE))
|
|
|
268 {
|
|
|
269 // This thread should do the initialization.
|
|
|
270 return 1;
|
|
|
271 }
|
|
|
272
|
|
|
273 if (expected == guard_bit)
|
|
|
274 {
|
|
|
275 // Already initialized.
|
|
|
276 return 0;
|
|
|
277 }
|
|
|
278
|
|
|
279 if (expected == pending_bit)
|
|
|
280 {
|
|
|
281 // Use acquire here.
|
|
|
282 int newv = expected | waiting_bit;
|
|
|
283 if (!__atomic_compare_exchange_n(gi, &expected, newv, false,
|
|
|
284 __ATOMIC_ACQ_REL,
|
|
|
285 __ATOMIC_ACQUIRE))
|
|
|
286 {
|
|
|
287 if (expected == guard_bit)
|
|
|
288 {
|
|
|
289 // Make a thread that failed to set the
|
|
|
290 // waiting bit exit the function earlier,
|
|
|
291 // if it detects that another thread has
|
|
|
292 // successfully finished initialising.
|
|
|
293 return 0;
|
|
|
294 }
|
|
|
295 if (expected == 0)
|
|
|
296 continue;
|
|
|
297 }
|
|
|
298
|
|
|
299 expected = newv;
|
|
|
300 }
|
|
|
301
|
|
|
302 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0);
|
|
|
303 }
|
|
|
304 }
|
|
|
305 # else
|
|
|
306 if (__gthread_active_p ())
|
|
|
307 {
|
|
|
308 mutex_wrapper mw;
|
|
|
309
|
|
|
310 while (1) // When this loop is executing, mutex is locked.
|
|
|
311 {
|
|
|
312 # ifdef __GTHREAD_HAS_COND
|
|
|
313 // The static is already initialized.
|
|
|
314 if (_GLIBCXX_GUARD_TEST(g))
|
|
|
315 return 0; // The mutex will be unlocked via wrapper
|
|
|
316
|
|
|
317 if (init_in_progress_flag(g))
|
|
|
318 {
|
|
|
319 // The guarded static is currently being initialized by
|
|
|
320 // another thread, so we release mutex and wait for the
|
|
|
321 // condition variable. We will lock the mutex again after
|
|
|
322 // this.
|
|
|
323 get_static_cond().wait_recursive(&get_static_mutex());
|
|
|
324 }
|
|
|
325 else
|
|
|
326 {
|
|
|
327 set_init_in_progress_flag(g, 1);
|
|
|
328 return 1; // The mutex will be unlocked via wrapper.
|
|
|
329 }
|
|
|
330 # else
|
|
|
331 // This provides compatibility with older systems not supporting
|
|
|
332 // POSIX like condition variables.
|
|
|
333 if (acquire(g))
|
|
|
334 {
|
|
|
335 mw.unlock = false;
|
|
|
336 return 1; // The mutex still locked.
|
|
|
337 }
|
|
|
338 return 0; // The mutex will be unlocked via wrapper.
|
|
|
339 # endif
|
|
|
340 }
|
|
|
341 }
|
|
|
342 # endif
|
|
|
343 #endif
|
|
|
344
|
|
|
345 return acquire (g);
|
|
|
346 }
|
|
|
347
|
|
|
348 extern "C"
|
|
|
349 void __cxa_guard_abort (__guard *g) throw ()
|
|
|
350 {
|
|
|
351 #ifdef _GLIBCXX_USE_FUTEX
|
|
|
352 // If __atomic_* and futex syscall are supported, don't use any global
|
|
|
353 // mutex.
|
|
|
354 if (__gthread_active_p ())
|
|
|
355 {
|
|
|
356 int *gi = (int *) (void *) g;
|
|
|
357 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
|
358 int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL);
|
|
|
359
|
|
|
360 if ((old & waiting_bit) != 0)
|
|
|
361 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
|
|
|
362 return;
|
|
|
363 }
|
|
|
364 #elif defined(__GTHREAD_HAS_COND)
|
|
|
365 if (__gthread_active_p())
|
|
|
366 {
|
|
|
367 mutex_wrapper mw;
|
|
|
368
|
|
|
369 set_init_in_progress_flag(g, 0);
|
|
|
370
|
|
|
371 // If we abort, we still need to wake up all other threads waiting for
|
|
|
372 // the condition variable.
|
|
|
373 get_static_cond().broadcast();
|
|
|
374 return;
|
|
|
375 }
|
|
|
376 #endif
|
|
|
377
|
|
|
378 set_init_in_progress_flag(g, 0);
|
|
|
379 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
|
|
|
380 // This provides compatibility with older systems not supporting POSIX like
|
|
|
381 // condition variables.
|
|
|
382 if (__gthread_active_p ())
|
|
|
383 static_mutex->unlock();
|
|
|
384 #endif
|
|
|
385 }
|
|
|
386
|
|
|
387 extern "C"
|
|
|
388 void __cxa_guard_release (__guard *g) throw ()
|
|
|
389 {
|
|
|
390 #ifdef _GLIBCXX_USE_FUTEX
|
|
|
391 // If __atomic_* and futex syscall are supported, don't use any global
|
|
|
392 // mutex.
|
|
|
393 if (__gthread_active_p ())
|
|
|
394 {
|
|
|
395 int *gi = (int *) (void *) g;
|
|
|
396 const int guard_bit = _GLIBCXX_GUARD_BIT;
|
|
|
397 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
|
398 int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL);
|
|
|
399
|
|
|
400 if ((old & waiting_bit) != 0)
|
|
|
401 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
|
|
|
402 return;
|
|
|
403 }
|
|
|
404 #elif defined(__GTHREAD_HAS_COND)
|
|
|
405 if (__gthread_active_p())
|
|
|
406 {
|
|
|
407 mutex_wrapper mw;
|
|
|
408
|
|
|
409 set_init_in_progress_flag(g, 0);
|
|
|
410 _GLIBCXX_GUARD_SET_AND_RELEASE(g);
|
|
|
411
|
|
|
412 get_static_cond().broadcast();
|
|
|
413 return;
|
|
|
414 }
|
|
|
415 #endif
|
|
|
416
|
|
|
417 set_init_in_progress_flag(g, 0);
|
|
|
418 _GLIBCXX_GUARD_SET_AND_RELEASE (g);
|
|
|
419
|
|
|
420 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
|
|
|
421 // This provides compatibility with older systems not supporting POSIX like
|
|
|
422 // condition variables.
|
|
|
423 if (__gthread_active_p())
|
|
|
424 static_mutex->unlock();
|
|
|
425 #endif
|
|
|
426 }
|
|
|
427 }
|
