|
145
|
1 /* Copyright (C) 2008-2020 Free Software Foundation, Inc.
|
|
111
|
2 Contributed by Richard Henderson <rth@redhat.com>.
|
|
|
3
|
|
|
4 This file is part of the GNU Transactional Memory Library (libitm).
|
|
|
5
|
|
|
6 Libitm is free software; you can redistribute it and/or modify it
|
|
|
7 under the terms of the GNU General Public License as published by
|
|
|
8 the Free Software Foundation; either version 3 of the License, or
|
|
|
9 (at your option) any later version.
|
|
|
10
|
|
|
11 Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
|
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
14 more details.
|
|
|
15
|
|
|
16 Under Section 7 of GPL version 3, you are granted additional
|
|
|
17 permissions described in the GCC Runtime Library Exception, version
|
|
|
18 3.1, as published by the Free Software Foundation.
|
|
|
19
|
|
|
20 You should have received a copy of the GNU General Public License and
|
|
|
21 a copy of the GCC Runtime Library Exception along with this program;
|
|
|
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
23 <http://www.gnu.org/licenses/>. */
|
|
|
24
|
|
|
25 #include "libitm_i.h"
|
|
|
26 #include <pthread.h>
|
|
|
27
|
|
|
28
|
|
|
29 using namespace GTM;
|
|
|
30
|
|
|
31 #if !defined(HAVE_ARCH_GTM_THREAD) || !defined(HAVE_ARCH_GTM_THREAD_DISP)
|
|
|
32 extern __thread gtm_thread_tls _gtm_thr_tls;
|
|
|
33 #endif
|
|
|
34
|
|
|
35 // Put this at the start of a cacheline so that serial_lock's writers and
|
|
|
36 // htm_fastpath fields are on the same cacheline, so that HW transactions
|
|
|
37 // only have to pay one cacheline capacity to monitor both.
|
|
|
38 gtm_rwlock GTM::gtm_thread::serial_lock
|
|
|
39 __attribute__((aligned(HW_CACHELINE_SIZE)));
|
|
|
40 gtm_thread *GTM::gtm_thread::list_of_threads = 0;
|
|
|
41 unsigned GTM::gtm_thread::number_of_threads = 0;
|
|
|
42
|
|
|
43 /* ??? Move elsewhere when we figure out library initialization. */
|
|
|
44 uint64_t GTM::gtm_spin_count_var = 1000;
|
|
|
45
|
|
|
46 #ifdef HAVE_64BIT_SYNC_BUILTINS
|
|
|
47 static atomic<_ITM_transactionId_t> global_tid;
|
|
|
48 #else
|
|
|
49 static _ITM_transactionId_t global_tid;
|
|
|
50 static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
|
51 #endif
|
|
|
52
|
|
|
53
|
|
|
54 // Provides a on-thread-exit callback used to release per-thread data.
|
|
|
55 static pthread_key_t thr_release_key;
|
|
|
56 static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
|
|
|
57
|
|
|
58 /* Allocate a transaction structure. */
|
|
|
59 void *
|
|
|
60 GTM::gtm_thread::operator new (size_t s)
|
|
|
61 {
|
|
|
62 void *tx;
|
|
|
63
|
|
|
64 assert(s == sizeof(gtm_thread));
|
|
|
65
|
|
|
66 tx = xmalloc (sizeof (gtm_thread), true);
|
|
|
67 memset (tx, 0, sizeof (gtm_thread));
|
|
|
68
|
|
|
69 return tx;
|
|
|
70 }
|
|
|
71
|
|
|
72 /* Free the given transaction. Raises an error if the transaction is still
|
|
|
73 in use. */
|
|
|
74 void
|
|
|
75 GTM::gtm_thread::operator delete(void *tx)
|
|
|
76 {
|
|
|
77 free(tx);
|
|
|
78 }
|
|
|
79
|
|
|
80 static void
|
|
|
81 thread_exit_handler(void *)
|
|
|
82 {
|
|
|
83 gtm_thread *thr = gtm_thr();
|
|
|
84 if (thr)
|
|
|
85 delete thr;
|
|
|
86 set_gtm_thr(0);
|
|
|
87 }
|
|
|
88
|
|
|
89 static void
|
|
|
90 thread_exit_init()
|
|
|
91 {
|
|
|
92 if (pthread_key_create(&thr_release_key, thread_exit_handler))
|
|
|
93 GTM_fatal("Creating thread release TLS key failed.");
|
|
|
94 }
|
|
|
95
|
|
|
96
|
|
|
97 GTM::gtm_thread::~gtm_thread()
|
|
|
98 {
|
|
|
99 if (nesting > 0)
|
|
|
100 GTM_fatal("Thread exit while a transaction is still active.");
|
|
|
101
|
|
|
102 // Deregister this transaction.
|
|
|
103 serial_lock.write_lock ();
|
|
|
104 gtm_thread **prev = &list_of_threads;
|
|
|
105 for (; *prev; prev = &(*prev)->next_thread)
|
|
|
106 {
|
|
|
107 if (*prev == this)
|
|
|
108 {
|
|
|
109 *prev = (*prev)->next_thread;
|
|
|
110 break;
|
|
|
111 }
|
|
|
112 }
|
|
|
113 number_of_threads--;
|
|
|
114 number_of_threads_changed(number_of_threads + 1, number_of_threads);
|
|
|
115 serial_lock.write_unlock ();
|
|
|
116 }
|
|
|
117
|
|
|
118 GTM::gtm_thread::gtm_thread ()
|
|
|
119 {
|
|
|
120 // This object's memory has been set to zero by operator new, so no need
|
|
|
121 // to initialize any of the other primitive-type members that do not have
|
|
|
122 // constructors.
|
|
|
123 shared_state.store(-1, memory_order_relaxed);
|
|
|
124
|
|
|
125 // Register this transaction with the list of all threads' transactions.
|
|
|
126 serial_lock.write_lock ();
|
|
|
127 next_thread = list_of_threads;
|
|
|
128 list_of_threads = this;
|
|
|
129 number_of_threads++;
|
|
|
130 number_of_threads_changed(number_of_threads - 1, number_of_threads);
|
|
|
131 serial_lock.write_unlock ();
|
|
|
132
|
|
|
133 init_cpp_exceptions ();
|
|
|
134
|
|
|
135 if (pthread_once(&thr_release_once, thread_exit_init))
|
|
|
136 GTM_fatal("Initializing thread release TLS key failed.");
|
|
|
137 // Any non-null value is sufficient to trigger destruction of this
|
|
|
138 // transaction when the current thread terminates.
|
|
|
139 if (pthread_setspecific(thr_release_key, this))
|
|
|
140 GTM_fatal("Setting thread release TLS key failed.");
|
|
|
141 }
|
|
|
142
|
|
|
143 static inline uint32_t
|
|
|
144 choose_code_path(uint32_t prop, abi_dispatch *disp)
|
|
|
145 {
|
|
|
146 if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
|
|
|
147 return a_runUninstrumentedCode;
|
|
|
148 else
|
|
|
149 return a_runInstrumentedCode;
|
|
|
150 }
|
|
|
151
|
|
|
152 #ifdef TARGET_BEGIN_TRANSACTION_ATTRIBUTE
|
|
|
153 /* This macro can be used to define target specific attributes for this
|
|
|
154 function. For example, S/390 requires floating point to be disabled in
|
|
|
155 begin_transaction. */
|
|
|
156 TARGET_BEGIN_TRANSACTION_ATTRIBUTE
|
|
|
157 #endif
|
|
|
158 uint32_t
|
|
|
159 GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
|
|
|
160 {
|
|
|
161 static const _ITM_transactionId_t tid_block_size = 1 << 16;
|
|
|
162
|
|
|
163 gtm_thread *tx;
|
|
|
164 abi_dispatch *disp;
|
|
|
165 uint32_t ret;
|
|
|
166
|
|
|
167 // ??? pr_undoLogCode is not properly defined in the ABI. Are barriers
|
|
|
168 // omitted because they are not necessary (e.g., a transaction on thread-
|
|
|
169 // local data) or because the compiler thinks that some kind of global
|
|
|
170 // synchronization might perform better?
|
|
|
171 if (unlikely(prop & pr_undoLogCode))
|
|
|
172 GTM_fatal("pr_undoLogCode not supported");
|
|
|
173
|
|
|
174 #ifdef USE_HTM_FASTPATH
|
|
|
175 // HTM fastpath. Only chosen in the absence of transaction_cancel to allow
|
|
|
176 // using an uninstrumented code path.
|
|
|
177 // The fastpath is enabled only by dispatch_htm's method group, which uses
|
|
|
178 // serial-mode methods as fallback. Serial-mode transactions cannot execute
|
|
|
179 // concurrently with HW transactions because the latter monitor the serial
|
|
|
180 // lock's writer flag and thus abort if another thread is or becomes a
|
|
|
181 // serial transaction. Therefore, if the fastpath is enabled, then a
|
|
|
182 // transaction is not executing as a HW transaction iff the serial lock is
|
|
|
183 // write-locked. Also, HW transactions monitor the fastpath control
|
|
|
184 // variable, so that they will only execute if dispatch_htm is still the
|
|
|
185 // current method group. This allows us to use htm_fastpath and the serial
|
|
|
186 // lock's writers flag to reliable determine whether the current thread runs
|
|
|
187 // a HW transaction, and thus we do not need to maintain this information in
|
|
|
188 // per-thread state.
|
|
|
189 // If an uninstrumented code path is not available, we can still run
|
|
|
190 // instrumented code from a HW transaction because the HTM fastpath kicks
|
|
|
191 // in early in both begin and commit, and the transaction is not canceled.
|
|
|
192 // HW transactions might get requests to switch to serial-irrevocable mode,
|
|
|
193 // but these can be ignored because the HTM provides all necessary
|
|
|
194 // correctness guarantees. Transactions cannot detect whether they are
|
|
|
195 // indeed in serial mode, and HW transactions should never need serial mode
|
|
|
196 // for any internal changes (e.g., they never abort visibly to the STM code
|
|
|
197 // and thus do not trigger the standard retry handling).
|
|
|
198 #ifndef HTM_CUSTOM_FASTPATH
|
|
|
199 if (likely(serial_lock.get_htm_fastpath() && (prop & pr_hasNoAbort)))
|
|
|
200 {
|
|
|
201 // Note that the snapshot of htm_fastpath that we take here could be
|
|
|
202 // outdated, and a different method group than dispatch_htm may have
|
|
|
203 // been chosen in the meantime. Therefore, take care not not touch
|
|
|
204 // anything besides the serial lock, which is independent of method
|
|
|
205 // groups.
|
|
|
206 for (uint32_t t = serial_lock.get_htm_fastpath(); t; t--)
|
|
|
207 {
|
|
|
208 uint32_t ret = htm_begin();
|
|
|
209 if (htm_begin_success(ret))
|
|
|
210 {
|
|
|
211 // We are executing a transaction now.
|
|
|
212 // Monitor the writer flag in the serial-mode lock, and abort
|
|
|
213 // if there is an active or waiting serial-mode transaction.
|
|
|
214 // Also checks that htm_fastpath is still nonzero and thus
|
|
|
215 // HW transactions are allowed to run.
|
|
|
216 // Note that this can also happen due to an enclosing
|
|
|
217 // serial-mode transaction; we handle this case below.
|
|
|
218 if (unlikely(serial_lock.htm_fastpath_disabled()))
|
|
|
219 htm_abort();
|
|
|
220 else
|
|
|
221 // We do not need to set a_saveLiveVariables because of HTM.
|
|
|
222 return (prop & pr_uninstrumentedCode) ?
|
|
|
223 a_runUninstrumentedCode : a_runInstrumentedCode;
|
|
|
224 }
|
|
|
225 // The transaction has aborted. Don't retry if it's unlikely that
|
|
|
226 // retrying the transaction will be successful.
|
|
|
227 if (!htm_abort_should_retry(ret))
|
|
|
228 break;
|
|
|
229 // Check whether the HTM fastpath has been disabled.
|
|
|
230 if (!serial_lock.get_htm_fastpath())
|
|
|
231 break;
|
|
|
232 // Wait until any concurrent serial-mode transactions have finished.
|
|
|
233 // This is an empty critical section, but won't be elided.
|
|
|
234 if (serial_lock.htm_fastpath_disabled())
|
|
|
235 {
|
|
|
236 tx = gtm_thr();
|
|
|
237 if (unlikely(tx == NULL))
|
|
|
238 {
|
|
|
239 // See below.
|
|
|
240 tx = new gtm_thread();
|
|
|
241 set_gtm_thr(tx);
|
|
|
242 }
|
|
|
243 // Check whether there is an enclosing serial-mode transaction;
|
|
|
244 // if so, we just continue as a nested transaction and don't
|
|
|
245 // try to use the HTM fastpath. This case can happen when an
|
|
|
246 // outermost relaxed transaction calls unsafe code that starts
|
|
|
247 // a transaction.
|
|
|
248 if (tx->nesting > 0)
|
|
|
249 break;
|
|
|
250 // Another thread is running a serial-mode transaction. Wait.
|
|
|
251 serial_lock.read_lock(tx);
|
|
|
252 serial_lock.read_unlock(tx);
|
|
|
253 // TODO We should probably reset the retry count t here, unless
|
|
|
254 // we have retried so often that we should go serial to avoid
|
|
|
255 // starvation.
|
|
|
256 }
|
|
|
257 }
|
|
|
258 }
|
|
|
259 #else
|
|
|
260 // If we have a custom HTM fastpath in ITM_beginTransaction, we implement
|
|
|
261 // just the retry policy here. We communicate with the custom fastpath
|
|
|
262 // through additional property bits and return codes, and either transfer
|
|
|
263 // control back to the custom fastpath or run the fallback mechanism. The
|
|
|
264 // fastpath synchronization algorithm itself is the same.
|
|
|
265 // pr_HTMRetryableAbort states that a HW transaction started by the custom
|
|
|
266 // HTM fastpath aborted, and that we thus have to decide whether to retry
|
|
|
267 // the fastpath (returning a_tryHTMFastPath) or just proceed with the
|
|
|
268 // fallback method.
|
|
|
269 if (likely(serial_lock.get_htm_fastpath() && (prop & pr_HTMRetryableAbort)))
|
|
|
270 {
|
|
|
271 tx = gtm_thr();
|
|
|
272 if (unlikely(tx == NULL))
|
|
|
273 {
|
|
|
274 // See below.
|
|
|
275 tx = new gtm_thread();
|
|
|
276 set_gtm_thr(tx);
|
|
|
277 }
|
|
|
278 // If this is the first abort, reset the retry count. We abuse
|
|
|
279 // restart_total for the retry count, which is fine because our only
|
|
|
280 // other fallback will use serial transactions, which don't use
|
|
|
281 // restart_total but will reset it when committing.
|
|
|
282 if (!(prop & pr_HTMRetriedAfterAbort))
|
|
|
283 tx->restart_total = gtm_thread::serial_lock.get_htm_fastpath();
|
|
|
284
|
|
|
285 if (--tx->restart_total > 0)
|
|
|
286 {
|
|
|
287 // Wait until any concurrent serial-mode transactions have finished.
|
|
|
288 // Essentially the same code as above.
|
|
|
289 if (!serial_lock.get_htm_fastpath())
|
|
|
290 goto stop_custom_htm_fastpath;
|
|
|
291 if (serial_lock.htm_fastpath_disabled())
|
|
|
292 {
|
|
|
293 if (tx->nesting > 0)
|
|
|
294 goto stop_custom_htm_fastpath;
|
|
|
295 serial_lock.read_lock(tx);
|
|
|
296 serial_lock.read_unlock(tx);
|
|
|
297 }
|
|
|
298 // Let ITM_beginTransaction retry the custom HTM fastpath.
|
|
|
299 return a_tryHTMFastPath;
|
|
|
300 }
|
|
|
301 }
|
|
|
302 stop_custom_htm_fastpath:
|
|
|
303 #endif
|
|
|
304 #endif
|
|
|
305
|
|
|
306 tx = gtm_thr();
|
|
|
307 if (unlikely(tx == NULL))
|
|
|
308 {
|
|
|
309 // Create the thread object. The constructor will also set up automatic
|
|
|
310 // deletion on thread termination.
|
|
|
311 tx = new gtm_thread();
|
|
|
312 set_gtm_thr(tx);
|
|
|
313 }
|
|
|
314
|
|
|
315 if (tx->nesting > 0)
|
|
|
316 {
|
|
|
317 // This is a nested transaction.
|
|
|
318 // Check prop compatibility:
|
|
|
319 // The ABI requires pr_hasNoFloatUpdate, pr_hasNoVectorUpdate,
|
|
|
320 // pr_hasNoIrrevocable, pr_aWBarriersOmitted, pr_RaRBarriersOmitted, and
|
|
|
321 // pr_hasNoSimpleReads to hold for the full dynamic scope of a
|
|
|
322 // transaction. We could check that these are set for the nested
|
|
|
323 // transaction if they are also set for the parent transaction, but the
|
|
|
324 // ABI does not require these flags to be set if they could be set,
|
|
|
325 // so the check could be too strict.
|
|
|
326 // ??? For pr_readOnly, lexical or dynamic scope is unspecified.
|
|
|
327
|
|
|
328 if (prop & pr_hasNoAbort)
|
|
|
329 {
|
|
|
330 // We can use flat nesting, so elide this transaction.
|
|
|
331 if (!(prop & pr_instrumentedCode))
|
|
|
332 {
|
|
|
333 if (!(tx->state & STATE_SERIAL) ||
|
|
|
334 !(tx->state & STATE_IRREVOCABLE))
|
|
|
335 tx->serialirr_mode();
|
|
|
336 }
|
|
|
337 // Increment nesting level after checking that we have a method that
|
|
|
338 // allows us to continue.
|
|
|
339 tx->nesting++;
|
|
|
340 return choose_code_path(prop, abi_disp());
|
|
|
341 }
|
|
|
342
|
|
|
343 // The transaction might abort, so use closed nesting if possible.
|
|
|
344 // pr_hasNoAbort has lexical scope, so the compiler should really have
|
|
|
345 // generated an instrumented code path.
|
|
|
346 assert(prop & pr_instrumentedCode);
|
|
|
347
|
|
|
348 // Create a checkpoint of the current transaction.
|
|
|
349 gtm_transaction_cp *cp = tx->parent_txns.push();
|
|
|
350 cp->save(tx);
|
|
|
351 new (&tx->alloc_actions) aa_tree<uintptr_t, gtm_alloc_action>();
|
|
|
352
|
|
|
353 // Check whether the current method actually supports closed nesting.
|
|
|
354 // If we can switch to another one, do so.
|
|
|
355 // If not, we assume that actual aborts are infrequent, and rather
|
|
|
356 // restart in _ITM_abortTransaction when we really have to.
|
|
|
357 disp = abi_disp();
|
|
|
358 if (!disp->closed_nesting())
|
|
|
359 {
|
|
|
360 // ??? Should we elide the transaction if there is no alternative
|
|
|
361 // method that supports closed nesting? If we do, we need to set
|
|
|
362 // some flag to prevent _ITM_abortTransaction from aborting the
|
|
|
363 // wrong transaction (i.e., some parent transaction).
|
|
|
364 abi_dispatch *cn_disp = disp->closed_nesting_alternative();
|
|
|
365 if (cn_disp)
|
|
|
366 {
|
|
|
367 disp = cn_disp;
|
|
|
368 set_abi_disp(disp);
|
|
|
369 }
|
|
|
370 }
|
|
|
371 }
|
|
|
372 else
|
|
|
373 {
|
|
|
374 // Outermost transaction
|
|
|
375 disp = tx->decide_begin_dispatch (prop);
|
|
|
376 set_abi_disp (disp);
|
|
|
377 }
|
|
|
378
|
|
|
379 // Initialization that is common for outermost and nested transactions.
|
|
|
380 tx->prop = prop;
|
|
|
381 tx->nesting++;
|
|
|
382
|
|
|
383 tx->jb = *jb;
|
|
|
384
|
|
|
385 // As long as we have not exhausted a previously allocated block of TIDs,
|
|
|
386 // we can avoid an atomic operation on a shared cacheline.
|
|
|
387 if (tx->local_tid & (tid_block_size - 1))
|
|
|
388 tx->id = tx->local_tid++;
|
|
|
389 else
|
|
|
390 {
|
|
|
391 #ifdef HAVE_64BIT_SYNC_BUILTINS
|
|
|
392 // We don't really care which block of TIDs we get but only that we
|
|
|
393 // acquire one atomically; therefore, relaxed memory order is
|
|
|
394 // sufficient.
|
|
|
395 tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
|
|
|
396 tx->local_tid = tx->id + 1;
|
|
|
397 #else
|
|
|
398 pthread_mutex_lock (&global_tid_lock);
|
|
|
399 global_tid += tid_block_size;
|
|
|
400 tx->id = global_tid;
|
|
|
401 tx->local_tid = tx->id + 1;
|
|
|
402 pthread_mutex_unlock (&global_tid_lock);
|
|
|
403 #endif
|
|
|
404 }
|
|
|
405
|
|
|
406 // Log the number of uncaught exceptions if we might have to roll back this
|
|
|
407 // state.
|
|
|
408 if (tx->cxa_uncaught_count_ptr != 0)
|
|
|
409 tx->cxa_uncaught_count = *tx->cxa_uncaught_count_ptr;
|
|
|
410
|
|
|
411 // Run dispatch-specific restart code. Retry until we succeed.
|
|
|
412 GTM::gtm_restart_reason rr;
|
|
|
413 while ((rr = disp->begin_or_restart()) != NO_RESTART)
|
|
|
414 {
|
|
|
415 tx->decide_retry_strategy(rr);
|
|
|
416 disp = abi_disp();
|
|
|
417 }
|
|
|
418
|
|
|
419 // Determine the code path to run. Only irrevocable transactions cannot be
|
|
|
420 // restarted, so all other transactions need to save live variables.
|
|
|
421 ret = choose_code_path(prop, disp);
|
|
|
422 if (!(tx->state & STATE_IRREVOCABLE))
|
|
|
423 ret |= a_saveLiveVariables;
|
|
|
424 return ret;
|
|
|
425 }
|
|
|
426
|
|
|
427
|
|
|
428 void
|
|
|
429 GTM::gtm_transaction_cp::save(gtm_thread* tx)
|
|
|
430 {
|
|
|
431 // Save everything that we might have to restore on restarts or aborts.
|
|
|
432 jb = tx->jb;
|
|
|
433 undolog_size = tx->undolog.size();
|
|
131
|
434
|
|
|
435 /* FIXME! Assignment of an aatree like alloc_actions is unsafe; if either
|
|
|
436 *this or *tx is destroyed, the other ends up pointing to a freed node. */
|
|
|
437 #pragma GCC diagnostic warning "-Wdeprecated-copy"
|
|
111
|
438 alloc_actions = tx->alloc_actions;
|
|
131
|
439
|
|
111
|
440 user_actions_size = tx->user_actions.size();
|
|
|
441 id = tx->id;
|
|
|
442 prop = tx->prop;
|
|
|
443 cxa_catch_count = tx->cxa_catch_count;
|
|
|
444 cxa_uncaught_count = tx->cxa_uncaught_count;
|
|
|
445 disp = abi_disp();
|
|
|
446 nesting = tx->nesting;
|
|
|
447 }
|
|
|
448
|
|
|
449 void
|
|
|
450 GTM::gtm_transaction_cp::commit(gtm_thread* tx)
|
|
|
451 {
|
|
|
452 // Restore state that is not persistent across commits. Exception handling,
|
|
|
453 // information, nesting level, and any logs do not need to be restored on
|
|
|
454 // commits of nested transactions. Allocation actions must be committed
|
|
|
455 // before committing the snapshot.
|
|
|
456 tx->jb = jb;
|
|
|
457 tx->alloc_actions = alloc_actions;
|
|
|
458 tx->id = id;
|
|
|
459 tx->prop = prop;
|
|
|
460 }
|
|
|
461
|
|
|
462
|
|
|
463 void
|
|
|
464 GTM::gtm_thread::rollback (gtm_transaction_cp *cp, bool aborting)
|
|
|
465 {
|
|
|
466 // The undo log is special in that it used for both thread-local and shared
|
|
|
467 // data. Because of the latter, we have to roll it back before any
|
|
|
468 // dispatch-specific rollback (which handles synchronization with other
|
|
|
469 // transactions).
|
|
|
470 undolog.rollback (this, cp ? cp->undolog_size : 0);
|
|
|
471
|
|
|
472 // Perform dispatch-specific rollback.
|
|
|
473 abi_disp()->rollback (cp);
|
|
|
474
|
|
|
475 // Roll back all actions that are supposed to happen around the transaction.
|
|
|
476 rollback_user_actions (cp ? cp->user_actions_size : 0);
|
|
|
477 commit_allocations (true, (cp ? &cp->alloc_actions : 0));
|
|
|
478 revert_cpp_exceptions (cp);
|
|
|
479
|
|
|
480 if (cp)
|
|
|
481 {
|
|
|
482 // We do not yet handle restarts of nested transactions. To do that, we
|
|
|
483 // would have to restore some state (jb, id, prop, nesting) not to the
|
|
|
484 // checkpoint but to the transaction that was started from this
|
|
|
485 // checkpoint (e.g., nesting = cp->nesting + 1);
|
|
|
486 assert(aborting);
|
|
|
487 // Roll back the rest of the state to the checkpoint.
|
|
|
488 jb = cp->jb;
|
|
|
489 id = cp->id;
|
|
|
490 prop = cp->prop;
|
|
|
491 if (cp->disp != abi_disp())
|
|
|
492 set_abi_disp(cp->disp);
|
|
|
493 alloc_actions = cp->alloc_actions;
|
|
|
494 nesting = cp->nesting;
|
|
|
495 }
|
|
|
496 else
|
|
|
497 {
|
|
|
498 // Roll back to the outermost transaction.
|
|
|
499 // Restore the jump buffer and transaction properties, which we will
|
|
|
500 // need for the longjmp used to restart or abort the transaction.
|
|
|
501 if (parent_txns.size() > 0)
|
|
|
502 {
|
|
|
503 jb = parent_txns[0].jb;
|
|
|
504 id = parent_txns[0].id;
|
|
|
505 prop = parent_txns[0].prop;
|
|
|
506 }
|
|
|
507 // Reset the transaction. Do not reset this->state, which is handled by
|
|
|
508 // the callers. Note that if we are not aborting, we reset the
|
|
|
509 // transaction to the point after having executed begin_transaction
|
|
|
510 // (we will return from it), so the nesting level must be one, not zero.
|
|
|
511 nesting = (aborting ? 0 : 1);
|
|
|
512 parent_txns.clear();
|
|
|
513 }
|
|
|
514
|
|
|
515 if (this->eh_in_flight)
|
|
|
516 {
|
|
|
517 _Unwind_DeleteException ((_Unwind_Exception *) this->eh_in_flight);
|
|
|
518 this->eh_in_flight = NULL;
|
|
|
519 }
|
|
|
520 }
|
|
|
521
|
|
|
522 void ITM_REGPARM
|
|
|
523 _ITM_abortTransaction (_ITM_abortReason reason)
|
|
|
524 {
|
|
|
525 gtm_thread *tx = gtm_thr();
|
|
|
526
|
|
|
527 assert (reason == userAbort || reason == (userAbort | outerAbort));
|
|
|
528 assert ((tx->prop & pr_hasNoAbort) == 0);
|
|
|
529
|
|
|
530 if (tx->state & gtm_thread::STATE_IRREVOCABLE)
|
|
|
531 abort ();
|
|
|
532
|
|
|
533 // Roll back to innermost transaction.
|
|
|
534 if (tx->parent_txns.size() > 0 && !(reason & outerAbort))
|
|
|
535 {
|
|
|
536 // If the current method does not support closed nesting but we are
|
|
|
537 // nested and must only roll back the innermost transaction, then
|
|
|
538 // restart with a method that supports closed nesting.
|
|
|
539 abi_dispatch *disp = abi_disp();
|
|
|
540 if (!disp->closed_nesting())
|
|
|
541 tx->restart(RESTART_CLOSED_NESTING);
|
|
|
542
|
|
|
543 // The innermost transaction is a closed nested transaction.
|
|
|
544 gtm_transaction_cp *cp = tx->parent_txns.pop();
|
|
|
545 uint32_t longjmp_prop = tx->prop;
|
|
|
546 gtm_jmpbuf longjmp_jb = tx->jb;
|
|
|
547
|
|
|
548 tx->rollback (cp, true);
|
|
|
549
|
|
|
550 // Jump to nested transaction (use the saved jump buffer).
|
|
|
551 GTM_longjmp (a_abortTransaction | a_restoreLiveVariables,
|
|
|
552 &longjmp_jb, longjmp_prop);
|
|
|
553 }
|
|
|
554 else
|
|
|
555 {
|
|
|
556 // There is no nested transaction or an abort of the outermost
|
|
|
557 // transaction was requested, so roll back to the outermost transaction.
|
|
|
558 tx->rollback (0, true);
|
|
|
559
|
|
|
560 // Aborting an outermost transaction finishes execution of the whole
|
|
|
561 // transaction. Therefore, reset transaction state.
|
|
|
562 if (tx->state & gtm_thread::STATE_SERIAL)
|
|
|
563 gtm_thread::serial_lock.write_unlock ();
|
|
|
564 else
|
|
|
565 gtm_thread::serial_lock.read_unlock (tx);
|
|
|
566 tx->state = 0;
|
|
|
567
|
|
|
568 GTM_longjmp (a_abortTransaction | a_restoreLiveVariables,
|
|
|
569 &tx->jb, tx->prop);
|
|
|
570 }
|
|
|
571 }
|
|
|
572
|
|
|
573 bool
|
|
|
574 GTM::gtm_thread::trycommit ()
|
|
|
575 {
|
|
|
576 nesting--;
|
|
|
577
|
|
|
578 // Skip any real commit for elided transactions.
|
|
|
579 if (nesting > 0 && (parent_txns.size() == 0 ||
|
|
|
580 nesting > parent_txns[parent_txns.size() - 1].nesting))
|
|
|
581 return true;
|
|
|
582
|
|
|
583 if (nesting > 0)
|
|
|
584 {
|
|
|
585 // Commit of a closed-nested transaction. Remove one checkpoint and add
|
|
|
586 // any effects of this transaction to the parent transaction.
|
|
|
587 gtm_transaction_cp *cp = parent_txns.pop();
|
|
|
588 commit_allocations(false, &cp->alloc_actions);
|
|
|
589 cp->commit(this);
|
|
|
590 return true;
|
|
|
591 }
|
|
|
592
|
|
|
593 // Commit of an outermost transaction.
|
|
|
594 gtm_word priv_time = 0;
|
|
|
595 if (abi_disp()->trycommit (priv_time))
|
|
|
596 {
|
|
|
597 // The transaction is now finished but we will still access some shared
|
|
|
598 // data if we have to ensure privatization safety.
|
|
|
599 bool do_read_unlock = false;
|
|
|
600 if (state & gtm_thread::STATE_SERIAL)
|
|
|
601 {
|
|
|
602 gtm_thread::serial_lock.write_unlock ();
|
|
|
603 // There are no other active transactions, so there's no need to
|
|
|
604 // enforce privatization safety.
|
|
|
605 priv_time = 0;
|
|
|
606 }
|
|
|
607 else
|
|
|
608 {
|
|
|
609 // If we have to ensure privatization safety, we must not yet
|
|
|
610 // release the read lock and become inactive because (1) we still
|
|
|
611 // have to go through the list of all transactions, which can be
|
|
|
612 // modified by serial mode threads, and (2) we interpret each
|
|
|
613 // transactions' shared_state in the context of what we believe to
|
|
|
614 // be the current method group (and serial mode transactions can
|
|
|
615 // change the method group). Therefore, if we have to ensure
|
|
|
616 // privatization safety, delay becoming inactive but set a maximum
|
|
|
617 // snapshot time (we have committed and thus have an empty snapshot,
|
|
|
618 // so it will always be most recent). Use release MO so that this
|
|
|
619 // synchronizes with other threads observing our snapshot time.
|
|
|
620 if (priv_time)
|
|
|
621 {
|
|
|
622 do_read_unlock = true;
|
|
|
623 shared_state.store((~(typeof gtm_thread::shared_state)0) - 1,
|
|
|
624 memory_order_release);
|
|
|
625 }
|
|
|
626 else
|
|
|
627 gtm_thread::serial_lock.read_unlock (this);
|
|
|
628 }
|
|
|
629 state = 0;
|
|
|
630
|
|
|
631 // We can commit the undo log after dispatch-specific commit and after
|
|
|
632 // making the transaction inactive because we only have to reset
|
|
|
633 // gtm_thread state.
|
|
|
634 undolog.commit ();
|
|
|
635 // Reset further transaction state.
|
|
|
636 cxa_catch_count = 0;
|
|
|
637 restart_total = 0;
|
|
|
638
|
|
|
639 // Ensure privatization safety, if necessary.
|
|
|
640 if (priv_time)
|
|
|
641 {
|
|
|
642 // There must be a seq_cst fence between the following loads of the
|
|
|
643 // other transactions' shared_state and the dispatch-specific stores
|
|
|
644 // that signal updates by this transaction (e.g., lock
|
|
|
645 // acquisitions). This ensures that if we read prior to other
|
|
|
646 // reader transactions setting their shared_state to 0, then those
|
|
|
647 // readers will observe our updates. We can reuse the seq_cst fence
|
|
|
648 // in serial_lock.read_unlock() if we performed that; if not, we
|
|
|
649 // issue the fence.
|
|
|
650 if (do_read_unlock)
|
|
|
651 atomic_thread_fence (memory_order_seq_cst);
|
|
|
652 // TODO Don't just spin but also block using cond vars / futexes
|
|
|
653 // here. Should probably be integrated with the serial lock code.
|
|
|
654 for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
|
|
|
655 it = it->next_thread)
|
|
|
656 {
|
|
|
657 if (it == this) continue;
|
|
|
658 // We need to load other threads' shared_state using acquire
|
|
|
659 // semantics (matching the release semantics of the respective
|
|
|
660 // updates). This is necessary to ensure that the other
|
|
|
661 // threads' memory accesses happen before our actions that
|
|
|
662 // assume privatization safety.
|
|
|
663 // TODO Are there any platform-specific optimizations (e.g.,
|
|
|
664 // merging barriers)?
|
|
|
665 while (it->shared_state.load(memory_order_acquire) < priv_time)
|
|
|
666 cpu_relax();
|
|
|
667 }
|
|
|
668 }
|
|
|
669
|
|
|
670 // After ensuring privatization safety, we are now truly inactive and
|
|
|
671 // thus can release the read lock. We will also execute potentially
|
|
|
672 // privatizing actions (e.g., calling free()). User actions are first.
|
|
|
673 if (do_read_unlock)
|
|
|
674 gtm_thread::serial_lock.read_unlock (this);
|
|
|
675 commit_user_actions ();
|
|
|
676 commit_allocations (false, 0);
|
|
|
677
|
|
|
678 return true;
|
|
|
679 }
|
|
|
680 return false;
|
|
|
681 }
|
|
|
682
|
|
|
683 void ITM_NORETURN
|
|
|
684 GTM::gtm_thread::restart (gtm_restart_reason r, bool finish_serial_upgrade)
|
|
|
685 {
|
|
|
686 // Roll back to outermost transaction. Do not reset transaction state because
|
|
|
687 // we will continue executing this transaction.
|
|
|
688 rollback ();
|
|
|
689
|
|
|
690 // If we have to restart while an upgrade of the serial lock is happening,
|
|
|
691 // we need to finish this here, after rollback (to ensure privatization
|
|
|
692 // safety despite undo writes) and before deciding about the retry strategy
|
|
|
693 // (which could switch to/from serial mode).
|
|
|
694 if (finish_serial_upgrade)
|
|
|
695 gtm_thread::serial_lock.write_upgrade_finish(this);
|
|
|
696
|
|
|
697 decide_retry_strategy (r);
|
|
|
698
|
|
|
699 // Run dispatch-specific restart code. Retry until we succeed.
|
|
|
700 abi_dispatch* disp = abi_disp();
|
|
|
701 GTM::gtm_restart_reason rr;
|
|
|
702 while ((rr = disp->begin_or_restart()) != NO_RESTART)
|
|
|
703 {
|
|
|
704 decide_retry_strategy(rr);
|
|
|
705 disp = abi_disp();
|
|
|
706 }
|
|
|
707
|
|
|
708 GTM_longjmp (choose_code_path(prop, disp) | a_restoreLiveVariables,
|
|
|
709 &jb, prop);
|
|
|
710 }
|
|
|
711
|
|
|
712 void ITM_REGPARM
|
|
|
713 _ITM_commitTransaction(void)
|
|
|
714 {
|
|
|
715 #if defined(USE_HTM_FASTPATH)
|
|
|
716 // HTM fastpath. If we are not executing a HW transaction, then we will be
|
|
|
717 // a serial-mode transaction. If we are, then there will be no other
|
|
|
718 // concurrent serial-mode transaction.
|
|
|
719 // See gtm_thread::begin_transaction.
|
|
|
720 if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
|
|
|
721 {
|
|
|
722 htm_commit();
|
|
|
723 return;
|
|
|
724 }
|
|
|
725 #endif
|
|
|
726 gtm_thread *tx = gtm_thr();
|
|
|
727 if (!tx->trycommit ())
|
|
|
728 tx->restart (RESTART_VALIDATE_COMMIT);
|
|
|
729 }
|
|
|
730
|
|
|
731 void ITM_REGPARM
|
|
|
732 _ITM_commitTransactionEH(void *exc_ptr)
|
|
|
733 {
|
|
|
734 #if defined(USE_HTM_FASTPATH)
|
|
|
735 // See _ITM_commitTransaction.
|
|
|
736 if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
|
|
|
737 {
|
|
|
738 htm_commit();
|
|
|
739 return;
|
|
|
740 }
|
|
|
741 #endif
|
|
|
742 gtm_thread *tx = gtm_thr();
|
|
|
743 if (!tx->trycommit ())
|
|
|
744 {
|
|
|
745 tx->eh_in_flight = exc_ptr;
|
|
|
746 tx->restart (RESTART_VALIDATE_COMMIT);
|
|
|
747 }
|
|
|
748 }
|
