|
145
|
1 /* Copyright (C) 2008-2020 Free Software Foundation, Inc.
|
|
111
|
2 Contributed by Richard Henderson <rth@redhat.com>.
|
|
|
3
|
|
|
4 This file is part of the GNU Transactional Memory Library (libitm).
|
|
|
5
|
|
|
6 Libitm is free software; you can redistribute it and/or modify it
|
|
|
7 under the terms of the GNU General Public License as published by
|
|
|
8 the Free Software Foundation; either version 3 of the License, or
|
|
|
9 (at your option) any later version.
|
|
|
10
|
|
|
11 Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
|
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
14 more details.
|
|
|
15
|
|
|
16 Under Section 7 of GPL version 3, you are granted additional
|
|
|
17 permissions described in the GCC Runtime Library Exception, version
|
|
|
18 3.1, as published by the Free Software Foundation.
|
|
|
19
|
|
|
20 You should have received a copy of the GNU General Public License and
|
|
|
21 a copy of the GCC Runtime Library Exception along with this program;
|
|
|
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
23 <http://www.gnu.org/licenses/>. */
|
|
|
24
|
|
|
25 /* The following are internal implementation functions and definitions.
|
|
|
26 To distinguish them from those defined by the Intel ABI, they all
|
|
|
27 begin with GTM/gtm. */
|
|
|
28
|
|
|
29 #ifndef LIBITM_I_H
|
|
|
30 #define LIBITM_I_H 1
|
|
|
31
|
|
|
32 #include "libitm.h"
|
|
|
33 #include "config.h"
|
|
|
34
|
|
|
35 #include <assert.h>
|
|
|
36 #include <stdlib.h>
|
|
|
37 #include <string.h>
|
|
|
38 #include <unwind.h>
|
|
|
39 #include "local_atomic"
|
|
|
40
|
|
|
41 /* Don't require libgcc_s.so for exceptions. */
|
|
|
42 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
|
|
|
43
|
|
|
44
|
|
|
45 #include "common.h"
|
|
|
46
|
|
|
47 namespace GTM HIDDEN {
|
|
|
48
|
|
|
49 using namespace std;
|
|
|
50
|
|
|
51 typedef unsigned int gtm_word __attribute__((mode (word)));
|
|
|
52
|
|
|
53 // These values are given to GTM_restart_transaction and indicate the
|
|
|
54 // reason for the restart. The reason is used to decide what STM
|
|
|
55 // implementation should be used during the next iteration.
|
|
|
56 enum gtm_restart_reason
|
|
|
57 {
|
|
|
58 RESTART_REALLOCATE,
|
|
|
59 RESTART_LOCKED_READ,
|
|
|
60 RESTART_LOCKED_WRITE,
|
|
|
61 RESTART_VALIDATE_READ,
|
|
|
62 RESTART_VALIDATE_WRITE,
|
|
|
63 RESTART_VALIDATE_COMMIT,
|
|
|
64 RESTART_SERIAL_IRR,
|
|
|
65 RESTART_NOT_READONLY,
|
|
|
66 RESTART_CLOSED_NESTING,
|
|
|
67 RESTART_INIT_METHOD_GROUP,
|
|
|
68 NUM_RESTARTS,
|
|
|
69 NO_RESTART = NUM_RESTARTS
|
|
|
70 };
|
|
|
71
|
|
|
72 } // namespace GTM
|
|
|
73
|
|
|
74 #include "target.h"
|
|
|
75 #include "rwlock.h"
|
|
|
76 #include "aatree.h"
|
|
|
77 #include "dispatch.h"
|
|
|
78 #include "containers.h"
|
|
|
79
|
|
|
80 #ifdef __USER_LABEL_PREFIX__
|
|
|
81 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
|
|
|
82 # define UPFX1(t) UPFX2(t)
|
|
|
83 # define UPFX2(t) #t
|
|
|
84 #else
|
|
|
85 # define UPFX
|
|
|
86 #endif
|
|
|
87
|
|
|
88 namespace GTM HIDDEN {
|
|
|
89
|
|
|
90 // A log of (de)allocation actions. We defer handling of some actions until
|
|
|
91 // a commit of the outermost transaction. We also rely on potentially having
|
|
|
92 // both an allocation and a deallocation for the same piece of memory in the
|
|
|
93 // log; the order in which such entries are processed does not matter because
|
|
|
94 // the actions are not in conflict (see below).
|
|
|
95 // This type is private to alloc.c, but needs to be defined so that
|
|
|
96 // the template used inside gtm_thread can instantiate.
|
|
|
97 struct gtm_alloc_action
|
|
|
98 {
|
|
|
99 // Iff free_fn_sz is nonzero, it must be used instead of free_fn, and vice
|
|
|
100 // versa.
|
|
|
101 void (*free_fn)(void *);
|
|
|
102 void (*free_fn_sz)(void *, size_t);
|
|
|
103 size_t sz;
|
|
|
104 // If true, this is an allocation; we discard the log entry on outermost
|
|
|
105 // commit, and deallocate on abort. If false, this is a deallocation and
|
|
|
106 // we deallocate on outermost commit and discard the log entry on abort.
|
|
|
107 bool allocated;
|
|
|
108 };
|
|
|
109
|
|
|
110 struct gtm_thread;
|
|
|
111
|
|
|
112 // A transaction checkpoint: data that has to saved and restored when doing
|
|
|
113 // closed nesting.
|
|
|
114 struct gtm_transaction_cp
|
|
|
115 {
|
|
|
116 gtm_jmpbuf jb;
|
|
|
117 size_t undolog_size;
|
|
|
118 aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
|
|
|
119 size_t user_actions_size;
|
|
|
120 _ITM_transactionId_t id;
|
|
|
121 uint32_t prop;
|
|
|
122 uint32_t cxa_catch_count;
|
|
|
123 unsigned int cxa_uncaught_count;
|
|
|
124 // We might want to use a different but compatible dispatch method for
|
|
|
125 // a nested transaction.
|
|
|
126 abi_dispatch *disp;
|
|
|
127 // Nesting level of this checkpoint (1 means that this is a checkpoint of
|
|
|
128 // the outermost transaction).
|
|
|
129 uint32_t nesting;
|
|
|
130
|
|
|
131 void save(gtm_thread* tx);
|
|
|
132 void commit(gtm_thread* tx);
|
|
|
133 };
|
|
|
134
|
|
|
135 // An undo log for writes.
|
|
|
136 struct gtm_undolog
|
|
|
137 {
|
|
|
138 vector<gtm_word> undolog;
|
|
|
139
|
|
|
140 // Log the previous value at a certain address.
|
|
|
141 // The easiest way to inline this is to just define this here.
|
|
|
142 void log(const void *ptr, size_t len)
|
|
|
143 {
|
|
|
144 size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
|
|
|
145 gtm_word *undo = undolog.push(words + 2);
|
|
|
146 memcpy(undo, ptr, len);
|
|
|
147 undo[words] = len;
|
|
|
148 undo[words + 1] = (gtm_word) ptr;
|
|
|
149 }
|
|
|
150
|
|
|
151 void commit () { undolog.clear(); }
|
|
|
152 size_t size() const { return undolog.size(); }
|
|
|
153
|
|
|
154 // In local.cc
|
|
|
155 void rollback (gtm_thread* tx, size_t until_size = 0);
|
|
|
156 };
|
|
|
157
|
|
|
158 // An entry of a read or write log. Used by multi-lock TM methods.
|
|
|
159 struct gtm_rwlog_entry
|
|
|
160 {
|
|
|
161 atomic<gtm_word> *orec;
|
|
|
162 gtm_word value;
|
|
|
163 };
|
|
|
164
|
|
|
165 // Contains all thread-specific data required by the entire library.
|
|
|
166 // This includes all data relevant to a single transaction. Because most
|
|
|
167 // thread-specific data is about the current transaction, we also refer to
|
|
|
168 // the transaction-specific parts of gtm_thread as "the transaction" (the
|
|
|
169 // same applies to names of variables and arguments).
|
|
|
170 // All but the shared part of this data structure are thread-local data.
|
|
|
171 // gtm_thread could be split into transaction-specific structures and other
|
|
|
172 // per-thread data (with those parts then nested in gtm_thread), but this
|
|
|
173 // would make it harder to later rearrange individual members to optimize data
|
|
|
174 // accesses. Thus, for now we keep one flat object, and will only split it if
|
|
|
175 // the code gets too messy.
|
|
|
176 struct gtm_thread
|
|
|
177 {
|
|
|
178
|
|
|
179 struct user_action
|
|
|
180 {
|
|
|
181 _ITM_userCommitFunction fn;
|
|
|
182 void *arg;
|
|
|
183 bool on_commit;
|
|
|
184 _ITM_transactionId_t resuming_id;
|
|
|
185 };
|
|
|
186
|
|
|
187 // The jump buffer by which GTM_longjmp restarts the transaction.
|
|
|
188 // This field *must* be at the beginning of the transaction.
|
|
|
189 gtm_jmpbuf jb;
|
|
|
190
|
|
|
191 // Data used by local.c for the undo log for both local and shared memory.
|
|
|
192 gtm_undolog undolog;
|
|
|
193
|
|
|
194 // Read and write logs. Used by multi-lock TM methods.
|
|
|
195 vector<gtm_rwlog_entry> readlog;
|
|
|
196 vector<gtm_rwlog_entry> writelog;
|
|
|
197
|
|
|
198 // Data used by alloc.c for the malloc/free undo log.
|
|
|
199 aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
|
|
|
200
|
|
|
201 // Data used by useraction.c for the user-defined commit/abort handlers.
|
|
|
202 vector<user_action> user_actions;
|
|
|
203
|
|
|
204 // A numerical identifier for this transaction.
|
|
|
205 _ITM_transactionId_t id;
|
|
|
206
|
|
|
207 // The _ITM_codeProperties of this transaction as given by the compiler.
|
|
|
208 uint32_t prop;
|
|
|
209
|
|
|
210 // The nesting depth for subsequently started transactions. This variable
|
|
|
211 // will be set to 1 when starting an outermost transaction.
|
|
|
212 uint32_t nesting;
|
|
|
213
|
|
|
214 // Set if this transaction owns the serial write lock.
|
|
|
215 // Can be reset only when restarting the outermost transaction.
|
|
|
216 static const uint32_t STATE_SERIAL = 0x0001;
|
|
|
217 // Set if the serial-irrevocable dispatch table is installed.
|
|
|
218 // Implies that no logging is being done, and abort is not possible.
|
|
|
219 // Can be reset only when restarting the outermost transaction.
|
|
|
220 static const uint32_t STATE_IRREVOCABLE = 0x0002;
|
|
|
221
|
|
|
222 // A bitmask of the above.
|
|
|
223 uint32_t state;
|
|
|
224
|
|
|
225 // In order to reduce cacheline contention on global_tid during
|
|
|
226 // beginTransaction, we allocate a block of 2**N ids to the thread
|
|
|
227 // all at once. This number is the next value to be allocated from
|
|
|
228 // the block, or 0 % 2**N if no such block is allocated.
|
|
|
229 _ITM_transactionId_t local_tid;
|
|
|
230
|
|
|
231 // Data used by eh_cpp.c for managing exceptions within the transaction.
|
|
|
232 uint32_t cxa_catch_count;
|
|
|
233 // If cxa_uncaught_count_ptr is 0, we don't need to roll back exceptions.
|
|
|
234 unsigned int *cxa_uncaught_count_ptr;
|
|
|
235 unsigned int cxa_uncaught_count;
|
|
|
236 void *eh_in_flight;
|
|
|
237
|
|
|
238 // Checkpoints for closed nesting.
|
|
|
239 vector<gtm_transaction_cp> parent_txns;
|
|
|
240
|
|
|
241 // Data used by retry.c for deciding what STM implementation should
|
|
|
242 // be used for the next iteration of the transaction.
|
|
|
243 // Only restart_total is reset to zero when the transaction commits, the
|
|
|
244 // other counters are total values for all previously executed transactions.
|
|
|
245 // restart_total is also used by the HTM fastpath in a different way.
|
|
|
246 uint32_t restart_reason[NUM_RESTARTS];
|
|
|
247 uint32_t restart_total;
|
|
|
248
|
|
|
249 // *** The shared part of gtm_thread starts here. ***
|
|
|
250 // Shared state is on separate cachelines to avoid false sharing with
|
|
|
251 // thread-local parts of gtm_thread.
|
|
|
252
|
|
|
253 // Points to the next thread in the list of all threads.
|
|
|
254 gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
|
|
|
255
|
|
|
256 // If this transaction is inactive, shared_state is ~0. Otherwise, this is
|
|
|
257 // an active or serial transaction.
|
|
|
258 atomic<gtm_word> shared_state;
|
|
|
259
|
|
|
260 // The lock that provides access to serial mode. Non-serialized
|
|
|
261 // transactions acquire read locks; a serialized transaction acquires
|
|
|
262 // a write lock.
|
|
|
263 // Accessed from assembly language, thus the "asm" specifier on
|
|
|
264 // the name, avoiding complex name mangling.
|
|
|
265 static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
|
|
|
266
|
|
|
267 // The head of the list of all threads' transactions.
|
|
|
268 static gtm_thread *list_of_threads;
|
|
|
269 // The number of all registered threads.
|
|
|
270 static unsigned number_of_threads;
|
|
|
271
|
|
|
272 // In alloc.cc
|
|
|
273 void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
|
|
|
274 void record_allocation (void *, void (*)(void *));
|
|
|
275 void forget_allocation (void *, void (*)(void *));
|
|
|
276 void forget_allocation (void *, size_t, void (*)(void *, size_t));
|
|
|
277 void discard_allocation (const void *ptr)
|
|
|
278 {
|
|
|
279 alloc_actions.erase((uintptr_t) ptr);
|
|
|
280 }
|
|
|
281
|
|
|
282 // In beginend.cc
|
|
|
283 void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
|
|
|
284 bool trycommit ();
|
|
|
285 void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
|
|
|
286 ITM_NORETURN;
|
|
|
287
|
|
|
288 gtm_thread();
|
|
|
289 ~gtm_thread();
|
|
|
290
|
|
|
291 static void *operator new(size_t);
|
|
|
292 static void operator delete(void *);
|
|
|
293
|
|
|
294 // Invoked from assembly language, thus the "asm" specifier on
|
|
|
295 // the name, avoiding complex name mangling.
|
|
|
296 static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
|
|
|
297 __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
|
|
|
298 // In eh_cpp.cc
|
|
|
299 void init_cpp_exceptions ();
|
|
|
300 void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
|
|
|
301
|
|
|
302 // In retry.cc
|
|
|
303 // Must be called outside of transactions (i.e., after rollback).
|
|
|
304 void decide_retry_strategy (gtm_restart_reason);
|
|
|
305 abi_dispatch* decide_begin_dispatch (uint32_t prop);
|
|
|
306 void number_of_threads_changed(unsigned previous, unsigned now);
|
|
|
307 // Must be called from serial mode. Does not call set_abi_disp().
|
|
|
308 void set_default_dispatch(abi_dispatch* disp);
|
|
|
309
|
|
|
310 // In method-serial.cc
|
|
|
311 void serialirr_mode ();
|
|
|
312
|
|
|
313 // In useraction.cc
|
|
|
314 void rollback_user_actions (size_t until_size = 0);
|
|
|
315 void commit_user_actions ();
|
|
|
316 };
|
|
|
317
|
|
|
318 } // namespace GTM
|
|
|
319
|
|
|
320 #include "tls.h"
|
|
|
321
|
|
|
322 namespace GTM HIDDEN {
|
|
|
323
|
|
|
324 // An unscaled count of the number of times we should spin attempting to
|
|
|
325 // acquire locks before we block the current thread and defer to the OS.
|
|
|
326 // This variable isn't used when the standard POSIX lock implementations
|
|
|
327 // are used.
|
|
|
328 extern uint64_t gtm_spin_count_var;
|
|
|
329
|
|
|
330 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
|
|
|
331 ITM_NORETURN ITM_REGPARM;
|
|
|
332
|
|
|
333 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
|
|
|
334
|
|
|
335 extern void GTM_error (const char *fmt, ...)
|
|
|
336 __attribute__((format (printf, 1, 2)));
|
|
|
337 extern void GTM_fatal (const char *fmt, ...)
|
|
|
338 __attribute__((noreturn, format (printf, 1, 2)));
|
|
|
339
|
|
|
340 extern abi_dispatch *dispatch_serial();
|
|
|
341 extern abi_dispatch *dispatch_serialirr();
|
|
|
342 extern abi_dispatch *dispatch_serialirr_onwrite();
|
|
|
343 extern abi_dispatch *dispatch_gl_wt();
|
|
|
344 extern abi_dispatch *dispatch_ml_wt();
|
|
|
345 extern abi_dispatch *dispatch_htm();
|
|
|
346
|
|
|
347
|
|
|
348 } // namespace GTM
|
|
|
349
|
|
|
350 #endif // LIBITM_I_H
|
