|
145
|
1 /* Copyright (C) 2017-2020 Free Software Foundation, Inc.
|
|
|
2
|
|
|
3 This file is part of GCC.
|
|
|
4
|
|
|
5 GCC is free software; you can redistribute it and/or modify it under
|
|
|
6 the terms of the GNU General Public License as published by the Free
|
|
|
7 Software Foundation; either version 3, or (at your option) any later
|
|
|
8 version.
|
|
|
9
|
|
|
10 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
11 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
13 for more details.
|
|
|
14
|
|
|
15 You should have received a copy of the GNU General Public License
|
|
|
16 along with GCC; see the file COPYING3. If not see
|
|
|
17 <http://www.gnu.org/licenses/>. */
|
|
|
18
|
|
|
19 /* {{{ Includes. */
|
|
|
20
|
|
|
21 #include "config.h"
|
|
|
22 #include "system.h"
|
|
|
23 #include "coretypes.h"
|
|
|
24 #include "backend.h"
|
|
|
25 #include "target.h"
|
|
|
26 #include "tree.h"
|
|
|
27 #include "gimple.h"
|
|
|
28 #include "tree-pass.h"
|
|
|
29 #include "gimple-iterator.h"
|
|
|
30 #include "cfghooks.h"
|
|
|
31 #include "cfgloop.h"
|
|
|
32 #include "tm_p.h"
|
|
|
33 #include "stringpool.h"
|
|
|
34 #include "fold-const.h"
|
|
|
35 #include "varasm.h"
|
|
|
36 #include "omp-low.h"
|
|
|
37 #include "omp-general.h"
|
|
|
38 #include "internal-fn.h"
|
|
|
39 #include "tree-vrp.h"
|
|
|
40 #include "tree-ssanames.h"
|
|
|
41 #include "tree-ssa-operands.h"
|
|
|
42 #include "gimplify.h"
|
|
|
43 #include "tree-phinodes.h"
|
|
|
44 #include "cgraph.h"
|
|
|
45 #include "targhooks.h"
|
|
|
46 #include "langhooks-def.h"
|
|
|
47
|
|
|
48 /* }}} */
|
|
|
49 /* {{{ OMP GCN pass.
|
|
|
50
|
|
|
51 This pass is intended to make any GCN-specfic transformations to OpenMP
|
|
|
52 target regions.
|
|
|
53
|
|
|
54 At present, its only purpose is to convert some "omp" built-in functions
|
|
|
55 to use closer-to-the-metal "gcn" built-in functions. */
|
|
|
56
|
|
|
57 unsigned int
|
|
|
58 execute_omp_gcn (void)
|
|
|
59 {
|
|
|
60 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
|
|
|
61 tree thr_num_id = DECL_NAME (thr_num_tree);
|
|
|
62 tree team_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
|
|
|
63 tree team_num_id = DECL_NAME (team_num_tree);
|
|
|
64 basic_block bb;
|
|
|
65 gimple_stmt_iterator gsi;
|
|
|
66 unsigned int todo = 0;
|
|
|
67
|
|
|
68 FOR_EACH_BB_FN (bb, cfun)
|
|
|
69 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
70 {
|
|
|
71 gimple *call = gsi_stmt (gsi);
|
|
|
72 tree decl;
|
|
|
73
|
|
|
74 if (is_gimple_call (call) && (decl = gimple_call_fndecl (call)))
|
|
|
75 {
|
|
|
76 tree decl_id = DECL_NAME (decl);
|
|
|
77 tree lhs = gimple_get_lhs (call);
|
|
|
78
|
|
|
79 if (decl_id == thr_num_id)
|
|
|
80 {
|
|
|
81 if (dump_file && (dump_flags & TDF_DETAILS))
|
|
|
82 fprintf (dump_file,
|
|
|
83 "Replace '%s' with __builtin_gcn_dim_pos.\n",
|
|
|
84 IDENTIFIER_POINTER (decl_id));
|
|
|
85
|
|
|
86 /* Transform this:
|
|
|
87 lhs = __builtin_omp_get_thread_num ()
|
|
|
88 to this:
|
|
|
89 lhs = __builtin_gcn_dim_pos (1) */
|
|
|
90 tree fn = targetm.builtin_decl (GCN_BUILTIN_OMP_DIM_POS, 0);
|
|
|
91 tree fnarg = build_int_cst (unsigned_type_node, 1);
|
|
|
92 gimple *stmt = gimple_build_call (fn, 1, fnarg);
|
|
|
93 gimple_call_set_lhs (stmt, lhs);
|
|
|
94 gsi_replace (&gsi, stmt, true);
|
|
|
95
|
|
|
96 todo |= TODO_update_ssa;
|
|
|
97 }
|
|
|
98 else if (decl_id == team_num_id)
|
|
|
99 {
|
|
|
100 if (dump_file && (dump_flags & TDF_DETAILS))
|
|
|
101 fprintf (dump_file,
|
|
|
102 "Replace '%s' with __builtin_gcn_dim_pos.\n",
|
|
|
103 IDENTIFIER_POINTER (decl_id));
|
|
|
104
|
|
|
105 /* Transform this:
|
|
|
106 lhs = __builtin_omp_get_team_num ()
|
|
|
107 to this:
|
|
|
108 lhs = __builtin_gcn_dim_pos (0) */
|
|
|
109 tree fn = targetm.builtin_decl (GCN_BUILTIN_OMP_DIM_POS, 0);
|
|
|
110 tree fnarg = build_zero_cst (unsigned_type_node);
|
|
|
111 gimple *stmt = gimple_build_call (fn, 1, fnarg);
|
|
|
112 gimple_call_set_lhs (stmt, lhs);
|
|
|
113 gsi_replace (&gsi, stmt, true);
|
|
|
114
|
|
|
115 todo |= TODO_update_ssa;
|
|
|
116 }
|
|
|
117 }
|
|
|
118 }
|
|
|
119
|
|
|
120 return todo;
|
|
|
121 }
|
|
|
122
|
|
|
123 namespace
|
|
|
124 {
|
|
|
125
|
|
|
126 const pass_data pass_data_omp_gcn = {
|
|
|
127 GIMPLE_PASS,
|
|
|
128 "omp_gcn", /* name */
|
|
|
129 OPTGROUP_NONE, /* optinfo_flags */
|
|
|
130 TV_NONE, /* tv_id */
|
|
|
131 0, /* properties_required */
|
|
|
132 0, /* properties_provided */
|
|
|
133 0, /* properties_destroyed */
|
|
|
134 0, /* todo_flags_start */
|
|
|
135 TODO_df_finish, /* todo_flags_finish */
|
|
|
136 };
|
|
|
137
|
|
|
138 class pass_omp_gcn : public gimple_opt_pass
|
|
|
139 {
|
|
|
140 public:
|
|
|
141 pass_omp_gcn (gcc::context *ctxt)
|
|
|
142 : gimple_opt_pass (pass_data_omp_gcn, ctxt)
|
|
|
143 {
|
|
|
144 }
|
|
|
145
|
|
|
146 /* opt_pass methods: */
|
|
|
147 virtual bool gate (function *)
|
|
|
148 {
|
|
|
149 return flag_openmp;
|
|
|
150 }
|
|
|
151
|
|
|
152 virtual unsigned int execute (function *)
|
|
|
153 {
|
|
|
154 return execute_omp_gcn ();
|
|
|
155 }
|
|
|
156
|
|
|
157 }; /* class pass_omp_gcn. */
|
|
|
158
|
|
|
159 } /* anon namespace. */
|
|
|
160
|
|
|
161 gimple_opt_pass *
|
|
|
162 make_pass_omp_gcn (gcc::context *ctxt)
|
|
|
163 {
|
|
|
164 return new pass_omp_gcn (ctxt);
|
|
|
165 }
|
|
|
166
|
|
|
167 /* }}} */
|
|
|
168 /* {{{ OpenACC reductions. */
|
|
|
169
|
|
|
170 /* Global lock variable, needed for 128bit worker & gang reductions. */
|
|
|
171
|
|
|
172 static GTY(()) tree global_lock_var;
|
|
|
173
|
|
|
174 /* Lazily generate the global_lock_var decl and return its address. */
|
|
|
175
|
|
|
176 static tree
|
|
|
177 gcn_global_lock_addr ()
|
|
|
178 {
|
|
|
179 tree v = global_lock_var;
|
|
|
180
|
|
|
181 if (!v)
|
|
|
182 {
|
|
|
183 tree name = get_identifier ("__reduction_lock");
|
|
|
184 tree type = build_qualified_type (unsigned_type_node,
|
|
|
185 TYPE_QUAL_VOLATILE);
|
|
|
186 v = build_decl (BUILTINS_LOCATION, VAR_DECL, name, type);
|
|
|
187 global_lock_var = v;
|
|
|
188 DECL_ARTIFICIAL (v) = 1;
|
|
|
189 DECL_EXTERNAL (v) = 1;
|
|
|
190 TREE_STATIC (v) = 1;
|
|
|
191 TREE_PUBLIC (v) = 1;
|
|
|
192 TREE_USED (v) = 1;
|
|
|
193 mark_addressable (v);
|
|
|
194 mark_decl_referenced (v);
|
|
|
195 }
|
|
|
196
|
|
|
197 return build_fold_addr_expr (v);
|
|
|
198 }
|
|
|
199
|
|
|
200 /* Helper function for gcn_reduction_update.
|
|
|
201
|
|
|
202 Insert code to locklessly update *PTR with *PTR OP VAR just before
|
|
|
203 GSI. We use a lockless scheme for nearly all case, which looks
|
|
|
204 like:
|
|
|
205 actual = initval (OP);
|
|
|
206 do {
|
|
|
207 guess = actual;
|
|
|
208 write = guess OP myval;
|
|
|
209 actual = cmp&swap (ptr, guess, write)
|
|
|
210 } while (actual bit-different-to guess);
|
|
|
211 return write;
|
|
|
212
|
|
|
213 This relies on a cmp&swap instruction, which is available for 32- and
|
|
|
214 64-bit types. Larger types must use a locking scheme. */
|
|
|
215
|
|
|
216 static tree
|
|
|
217 gcn_lockless_update (location_t loc, gimple_stmt_iterator *gsi,
|
|
|
218 tree ptr, tree var, tree_code op)
|
|
|
219 {
|
|
|
220 unsigned fn = GCN_BUILTIN_CMP_SWAP;
|
|
|
221 tree_code code = NOP_EXPR;
|
|
|
222 tree arg_type = unsigned_type_node;
|
|
|
223 tree var_type = TREE_TYPE (var);
|
|
|
224
|
|
|
225 if (TREE_CODE (var_type) == COMPLEX_TYPE
|
|
|
226 || TREE_CODE (var_type) == REAL_TYPE)
|
|
|
227 code = VIEW_CONVERT_EXPR;
|
|
|
228
|
|
|
229 if (TYPE_SIZE (var_type) == TYPE_SIZE (long_long_unsigned_type_node))
|
|
|
230 {
|
|
|
231 arg_type = long_long_unsigned_type_node;
|
|
|
232 fn = GCN_BUILTIN_CMP_SWAPLL;
|
|
|
233 }
|
|
|
234
|
|
|
235 tree swap_fn = gcn_builtin_decl (fn, true);
|
|
|
236
|
|
|
237 gimple_seq init_seq = NULL;
|
|
|
238 tree init_var = make_ssa_name (arg_type);
|
|
|
239 tree init_expr = omp_reduction_init_op (loc, op, var_type);
|
|
|
240 init_expr = fold_build1 (code, arg_type, init_expr);
|
|
|
241 gimplify_assign (init_var, init_expr, &init_seq);
|
|
|
242 gimple *init_end = gimple_seq_last (init_seq);
|
|
|
243
|
|
|
244 gsi_insert_seq_before (gsi, init_seq, GSI_SAME_STMT);
|
|
|
245
|
|
|
246 /* Split the block just after the init stmts. */
|
|
|
247 basic_block pre_bb = gsi_bb (*gsi);
|
|
|
248 edge pre_edge = split_block (pre_bb, init_end);
|
|
|
249 basic_block loop_bb = pre_edge->dest;
|
|
|
250 pre_bb = pre_edge->src;
|
|
|
251 /* Reset the iterator. */
|
|
|
252 *gsi = gsi_for_stmt (gsi_stmt (*gsi));
|
|
|
253
|
|
|
254 tree expect_var = make_ssa_name (arg_type);
|
|
|
255 tree actual_var = make_ssa_name (arg_type);
|
|
|
256 tree write_var = make_ssa_name (arg_type);
|
|
|
257
|
|
|
258 /* Build and insert the reduction calculation. */
|
|
|
259 gimple_seq red_seq = NULL;
|
|
|
260 tree write_expr = fold_build1 (code, var_type, expect_var);
|
|
|
261 write_expr = fold_build2 (op, var_type, write_expr, var);
|
|
|
262 write_expr = fold_build1 (code, arg_type, write_expr);
|
|
|
263 gimplify_assign (write_var, write_expr, &red_seq);
|
|
|
264
|
|
|
265 gsi_insert_seq_before (gsi, red_seq, GSI_SAME_STMT);
|
|
|
266
|
|
|
267 /* Build & insert the cmp&swap sequence. */
|
|
|
268 gimple_seq latch_seq = NULL;
|
|
|
269 tree swap_expr = build_call_expr_loc (loc, swap_fn, 3,
|
|
|
270 ptr, expect_var, write_var);
|
|
|
271 gimplify_assign (actual_var, swap_expr, &latch_seq);
|
|
|
272
|
|
|
273 gcond *cond = gimple_build_cond (EQ_EXPR, actual_var, expect_var,
|
|
|
274 NULL_TREE, NULL_TREE);
|
|
|
275 gimple_seq_add_stmt (&latch_seq, cond);
|
|
|
276
|
|
|
277 gimple *latch_end = gimple_seq_last (latch_seq);
|
|
|
278 gsi_insert_seq_before (gsi, latch_seq, GSI_SAME_STMT);
|
|
|
279
|
|
|
280 /* Split the block just after the latch stmts. */
|
|
|
281 edge post_edge = split_block (loop_bb, latch_end);
|
|
|
282 basic_block post_bb = post_edge->dest;
|
|
|
283 loop_bb = post_edge->src;
|
|
|
284 *gsi = gsi_for_stmt (gsi_stmt (*gsi));
|
|
|
285
|
|
|
286 post_edge->flags ^= EDGE_TRUE_VALUE | EDGE_FALLTHRU;
|
|
|
287 /* post_edge->probability = profile_probability::even (); */
|
|
|
288 edge loop_edge = make_edge (loop_bb, loop_bb, EDGE_FALSE_VALUE);
|
|
|
289 /* loop_edge->probability = profile_probability::even (); */
|
|
|
290 set_immediate_dominator (CDI_DOMINATORS, loop_bb, pre_bb);
|
|
|
291 set_immediate_dominator (CDI_DOMINATORS, post_bb, loop_bb);
|
|
|
292
|
|
|
293 gphi *phi = create_phi_node (expect_var, loop_bb);
|
|
|
294 add_phi_arg (phi, init_var, pre_edge, loc);
|
|
|
295 add_phi_arg (phi, actual_var, loop_edge, loc);
|
|
|
296
|
|
|
297 loop *loop = alloc_loop ();
|
|
|
298 loop->header = loop_bb;
|
|
|
299 loop->latch = loop_bb;
|
|
|
300 add_loop (loop, loop_bb->loop_father);
|
|
|
301
|
|
|
302 return fold_build1 (code, var_type, write_var);
|
|
|
303 }
|
|
|
304
|
|
|
305 /* Helper function for gcn_reduction_update.
|
|
|
306
|
|
|
307 Insert code to lockfully update *PTR with *PTR OP VAR just before
|
|
|
308 GSI. This is necessary for types larger than 64 bits, where there
|
|
|
309 is no cmp&swap instruction to implement a lockless scheme. We use
|
|
|
310 a lock variable in global memory.
|
|
|
311
|
|
|
312 while (cmp&swap (&lock_var, 0, 1))
|
|
|
313 continue;
|
|
|
314 T accum = *ptr;
|
|
|
315 accum = accum OP var;
|
|
|
316 *ptr = accum;
|
|
|
317 cmp&swap (&lock_var, 1, 0);
|
|
|
318 return accum;
|
|
|
319
|
|
|
320 A lock in global memory is necessary to force execution engine
|
|
|
321 descheduling and avoid resource starvation that can occur if the
|
|
|
322 lock is in shared memory. */
|
|
|
323
|
|
|
324 static tree
|
|
|
325 gcn_lockfull_update (location_t loc, gimple_stmt_iterator *gsi,
|
|
|
326 tree ptr, tree var, tree_code op)
|
|
|
327 {
|
|
|
328 tree var_type = TREE_TYPE (var);
|
|
|
329 tree swap_fn = gcn_builtin_decl (GCN_BUILTIN_CMP_SWAP, true);
|
|
|
330 tree uns_unlocked = build_int_cst (unsigned_type_node, 0);
|
|
|
331 tree uns_locked = build_int_cst (unsigned_type_node, 1);
|
|
|
332
|
|
|
333 /* Split the block just before the gsi. Insert a gimple nop to make
|
|
|
334 this easier. */
|
|
|
335 gimple *nop = gimple_build_nop ();
|
|
|
336 gsi_insert_before (gsi, nop, GSI_SAME_STMT);
|
|
|
337 basic_block entry_bb = gsi_bb (*gsi);
|
|
|
338 edge entry_edge = split_block (entry_bb, nop);
|
|
|
339 basic_block lock_bb = entry_edge->dest;
|
|
|
340 /* Reset the iterator. */
|
|
|
341 *gsi = gsi_for_stmt (gsi_stmt (*gsi));
|
|
|
342
|
|
|
343 /* Build and insert the locking sequence. */
|
|
|
344 gimple_seq lock_seq = NULL;
|
|
|
345 tree lock_var = make_ssa_name (unsigned_type_node);
|
|
|
346 tree lock_expr = gcn_global_lock_addr ();
|
|
|
347 lock_expr = build_call_expr_loc (loc, swap_fn, 3, lock_expr,
|
|
|
348 uns_unlocked, uns_locked);
|
|
|
349 gimplify_assign (lock_var, lock_expr, &lock_seq);
|
|
|
350 gcond *cond = gimple_build_cond (EQ_EXPR, lock_var, uns_unlocked,
|
|
|
351 NULL_TREE, NULL_TREE);
|
|
|
352 gimple_seq_add_stmt (&lock_seq, cond);
|
|
|
353 gimple *lock_end = gimple_seq_last (lock_seq);
|
|
|
354 gsi_insert_seq_before (gsi, lock_seq, GSI_SAME_STMT);
|
|
|
355
|
|
|
356 /* Split the block just after the lock sequence. */
|
|
|
357 edge locked_edge = split_block (lock_bb, lock_end);
|
|
|
358 basic_block update_bb = locked_edge->dest;
|
|
|
359 lock_bb = locked_edge->src;
|
|
|
360 *gsi = gsi_for_stmt (gsi_stmt (*gsi));
|
|
|
361
|
|
|
362 /* Create the lock loop. */
|
|
|
363 locked_edge->flags ^= EDGE_TRUE_VALUE | EDGE_FALLTHRU;
|
|
|
364 locked_edge->probability = profile_probability::even ();
|
|
|
365 edge loop_edge = make_edge (lock_bb, lock_bb, EDGE_FALSE_VALUE);
|
|
|
366 loop_edge->probability = profile_probability::even ();
|
|
|
367 set_immediate_dominator (CDI_DOMINATORS, lock_bb, entry_bb);
|
|
|
368 set_immediate_dominator (CDI_DOMINATORS, update_bb, lock_bb);
|
|
|
369
|
|
|
370 /* Create the loop structure. */
|
|
|
371 loop *lock_loop = alloc_loop ();
|
|
|
372 lock_loop->header = lock_bb;
|
|
|
373 lock_loop->latch = lock_bb;
|
|
|
374 lock_loop->nb_iterations_estimate = 1;
|
|
|
375 lock_loop->any_estimate = true;
|
|
|
376 add_loop (lock_loop, entry_bb->loop_father);
|
|
|
377
|
|
|
378 /* Build and insert the reduction calculation. */
|
|
|
379 gimple_seq red_seq = NULL;
|
|
|
380 tree acc_in = make_ssa_name (var_type);
|
|
|
381 tree ref_in = build_simple_mem_ref (ptr);
|
|
|
382 TREE_THIS_VOLATILE (ref_in) = 1;
|
|
|
383 gimplify_assign (acc_in, ref_in, &red_seq);
|
|
|
384
|
|
|
385 tree acc_out = make_ssa_name (var_type);
|
|
|
386 tree update_expr = fold_build2 (op, var_type, ref_in, var);
|
|
|
387 gimplify_assign (acc_out, update_expr, &red_seq);
|
|
|
388
|
|
|
389 tree ref_out = build_simple_mem_ref (ptr);
|
|
|
390 TREE_THIS_VOLATILE (ref_out) = 1;
|
|
|
391 gimplify_assign (ref_out, acc_out, &red_seq);
|
|
|
392
|
|
|
393 gsi_insert_seq_before (gsi, red_seq, GSI_SAME_STMT);
|
|
|
394
|
|
|
395 /* Build & insert the unlock sequence. */
|
|
|
396 gimple_seq unlock_seq = NULL;
|
|
|
397 tree unlock_expr = gcn_global_lock_addr ();
|
|
|
398 unlock_expr = build_call_expr_loc (loc, swap_fn, 3, unlock_expr,
|
|
|
399 uns_locked, uns_unlocked);
|
|
|
400 gimplify_and_add (unlock_expr, &unlock_seq);
|
|
|
401 gsi_insert_seq_before (gsi, unlock_seq, GSI_SAME_STMT);
|
|
|
402
|
|
|
403 return acc_out;
|
|
|
404 }
|
|
|
405
|
|
|
406 /* Emit a sequence to update a reduction accumulator at *PTR with the
|
|
|
407 value held in VAR using operator OP. Return the updated value.
|
|
|
408
|
|
|
409 TODO: optimize for atomic ops and independent complex ops. */
|
|
|
410
|
|
|
411 static tree
|
|
|
412 gcn_reduction_update (location_t loc, gimple_stmt_iterator *gsi,
|
|
|
413 tree ptr, tree var, tree_code op)
|
|
|
414 {
|
|
|
415 tree type = TREE_TYPE (var);
|
|
|
416 tree size = TYPE_SIZE (type);
|
|
|
417
|
|
|
418 if (size == TYPE_SIZE (unsigned_type_node)
|
|
|
419 || size == TYPE_SIZE (long_long_unsigned_type_node))
|
|
|
420 return gcn_lockless_update (loc, gsi, ptr, var, op);
|
|
|
421 else
|
|
|
422 return gcn_lockfull_update (loc, gsi, ptr, var, op);
|
|
|
423 }
|
|
|
424
|
|
|
425 /* Return a temporary variable decl to use for an OpenACC worker reduction. */
|
|
|
426
|
|
|
427 static tree
|
|
|
428 gcn_goacc_get_worker_red_decl (tree type, unsigned offset)
|
|
|
429 {
|
|
|
430 machine_function *machfun = cfun->machine;
|
|
|
431 tree existing_decl;
|
|
|
432
|
|
|
433 if (TREE_CODE (type) == REFERENCE_TYPE)
|
|
|
434 type = TREE_TYPE (type);
|
|
|
435
|
|
|
436 tree var_type
|
|
|
437 = build_qualified_type (type,
|
|
|
438 (TYPE_QUALS (type)
|
|
|
439 | ENCODE_QUAL_ADDR_SPACE (ADDR_SPACE_LDS)));
|
|
|
440
|
|
|
441 if (machfun->reduc_decls
|
|
|
442 && offset < machfun->reduc_decls->length ()
|
|
|
443 && (existing_decl = (*machfun->reduc_decls)[offset]))
|
|
|
444 {
|
|
|
445 gcc_assert (TREE_TYPE (existing_decl) == var_type);
|
|
|
446 return existing_decl;
|
|
|
447 }
|
|
|
448 else
|
|
|
449 {
|
|
|
450 char name[50];
|
|
|
451 sprintf (name, ".oacc_reduction_%u", offset);
|
|
|
452 tree decl = create_tmp_var_raw (var_type, name);
|
|
|
453
|
|
|
454 DECL_CONTEXT (decl) = NULL_TREE;
|
|
|
455 TREE_STATIC (decl) = 1;
|
|
|
456
|
|
|
457 varpool_node::finalize_decl (decl);
|
|
|
458
|
|
|
459 vec_safe_grow_cleared (machfun->reduc_decls, offset + 1);
|
|
|
460 (*machfun->reduc_decls)[offset] = decl;
|
|
|
461
|
|
|
462 return decl;
|
|
|
463 }
|
|
|
464
|
|
|
465 return NULL_TREE;
|
|
|
466 }
|
|
|
467
|
|
|
468 /* Expand IFN_GOACC_REDUCTION_SETUP. */
|
|
|
469
|
|
|
470 static void
|
|
|
471 gcn_goacc_reduction_setup (gcall *call)
|
|
|
472 {
|
|
|
473 gimple_stmt_iterator gsi = gsi_for_stmt (call);
|
|
|
474 tree lhs = gimple_call_lhs (call);
|
|
|
475 tree var = gimple_call_arg (call, 2);
|
|
|
476 int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
|
|
|
477 gimple_seq seq = NULL;
|
|
|
478
|
|
|
479 push_gimplify_context (true);
|
|
|
480
|
|
|
481 if (level != GOMP_DIM_GANG)
|
|
|
482 {
|
|
|
483 /* Copy the receiver object. */
|
|
|
484 tree ref_to_res = gimple_call_arg (call, 1);
|
|
|
485
|
|
|
486 if (!integer_zerop (ref_to_res))
|
|
|
487 var = build_simple_mem_ref (ref_to_res);
|
|
|
488 }
|
|
|
489
|
|
|
490 if (level == GOMP_DIM_WORKER)
|
|
|
491 {
|
|
|
492 tree var_type = TREE_TYPE (var);
|
|
|
493 /* Store incoming value to worker reduction buffer. */
|
|
|
494 tree offset = gimple_call_arg (call, 5);
|
|
|
495 tree decl
|
|
|
496 = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset));
|
|
|
497
|
|
|
498 gimplify_assign (decl, var, &seq);
|
|
|
499 }
|
|
|
500
|
|
|
501 if (lhs)
|
|
|
502 gimplify_assign (lhs, var, &seq);
|
|
|
503
|
|
|
504 pop_gimplify_context (NULL);
|
|
|
505 gsi_replace_with_seq (&gsi, seq, true);
|
|
|
506 }
|
|
|
507
|
|
|
508 /* Expand IFN_GOACC_REDUCTION_INIT. */
|
|
|
509
|
|
|
510 static void
|
|
|
511 gcn_goacc_reduction_init (gcall *call)
|
|
|
512 {
|
|
|
513 gimple_stmt_iterator gsi = gsi_for_stmt (call);
|
|
|
514 tree lhs = gimple_call_lhs (call);
|
|
|
515 tree var = gimple_call_arg (call, 2);
|
|
|
516 int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
|
|
|
517 enum tree_code rcode
|
|
|
518 = (enum tree_code) TREE_INT_CST_LOW (gimple_call_arg (call, 4));
|
|
|
519 tree init = omp_reduction_init_op (gimple_location (call), rcode,
|
|
|
520 TREE_TYPE (var));
|
|
|
521 gimple_seq seq = NULL;
|
|
|
522
|
|
|
523 push_gimplify_context (true);
|
|
|
524
|
|
|
525 if (level == GOMP_DIM_GANG)
|
|
|
526 {
|
|
|
527 /* If there's no receiver object, propagate the incoming VAR. */
|
|
|
528 tree ref_to_res = gimple_call_arg (call, 1);
|
|
|
529 if (integer_zerop (ref_to_res))
|
|
|
530 init = var;
|
|
|
531 }
|
|
|
532
|
|
|
533 if (lhs)
|
|
|
534 gimplify_assign (lhs, init, &seq);
|
|
|
535
|
|
|
536 pop_gimplify_context (NULL);
|
|
|
537 gsi_replace_with_seq (&gsi, seq, true);
|
|
|
538 }
|
|
|
539
|
|
|
540 /* Expand IFN_GOACC_REDUCTION_FINI. */
|
|
|
541
|
|
|
542 static void
|
|
|
543 gcn_goacc_reduction_fini (gcall *call)
|
|
|
544 {
|
|
|
545 gimple_stmt_iterator gsi = gsi_for_stmt (call);
|
|
|
546 tree lhs = gimple_call_lhs (call);
|
|
|
547 tree ref_to_res = gimple_call_arg (call, 1);
|
|
|
548 tree var = gimple_call_arg (call, 2);
|
|
|
549 int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
|
|
|
550 enum tree_code op
|
|
|
551 = (enum tree_code) TREE_INT_CST_LOW (gimple_call_arg (call, 4));
|
|
|
552 gimple_seq seq = NULL;
|
|
|
553 tree r = NULL_TREE;;
|
|
|
554
|
|
|
555 push_gimplify_context (true);
|
|
|
556
|
|
|
557 tree accum = NULL_TREE;
|
|
|
558
|
|
|
559 if (level == GOMP_DIM_WORKER)
|
|
|
560 {
|
|
|
561 tree var_type = TREE_TYPE (var);
|
|
|
562 tree offset = gimple_call_arg (call, 5);
|
|
|
563 tree decl
|
|
|
564 = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset));
|
|
|
565
|
|
|
566 accum = build_fold_addr_expr (decl);
|
|
|
567 }
|
|
|
568 else if (integer_zerop (ref_to_res))
|
|
|
569 r = var;
|
|
|
570 else
|
|
|
571 accum = ref_to_res;
|
|
|
572
|
|
|
573 if (accum)
|
|
|
574 {
|
|
|
575 /* UPDATE the accumulator. */
|
|
|
576 gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
|
|
|
577 seq = NULL;
|
|
|
578 r = gcn_reduction_update (gimple_location (call), &gsi, accum, var, op);
|
|
|
579 }
|
|
|
580
|
|
|
581 if (lhs)
|
|
|
582 gimplify_assign (lhs, r, &seq);
|
|
|
583 pop_gimplify_context (NULL);
|
|
|
584
|
|
|
585 gsi_replace_with_seq (&gsi, seq, true);
|
|
|
586 }
|
|
|
587
|
|
|
588 /* Expand IFN_GOACC_REDUCTION_TEARDOWN. */
|
|
|
589
|
|
|
590 static void
|
|
|
591 gcn_goacc_reduction_teardown (gcall *call)
|
|
|
592 {
|
|
|
593 gimple_stmt_iterator gsi = gsi_for_stmt (call);
|
|
|
594 tree lhs = gimple_call_lhs (call);
|
|
|
595 tree var = gimple_call_arg (call, 2);
|
|
|
596 int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
|
|
|
597 gimple_seq seq = NULL;
|
|
|
598
|
|
|
599 push_gimplify_context (true);
|
|
|
600
|
|
|
601 if (level == GOMP_DIM_WORKER)
|
|
|
602 {
|
|
|
603 tree var_type = TREE_TYPE (var);
|
|
|
604
|
|
|
605 /* Read the worker reduction buffer. */
|
|
|
606 tree offset = gimple_call_arg (call, 5);
|
|
|
607 tree decl
|
|
|
608 = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset));
|
|
|
609 var = decl;
|
|
|
610 }
|
|
|
611
|
|
|
612 if (level != GOMP_DIM_GANG)
|
|
|
613 {
|
|
|
614 /* Write to the receiver object. */
|
|
|
615 tree ref_to_res = gimple_call_arg (call, 1);
|
|
|
616
|
|
|
617 if (!integer_zerop (ref_to_res))
|
|
|
618 gimplify_assign (build_simple_mem_ref (ref_to_res), var, &seq);
|
|
|
619 }
|
|
|
620
|
|
|
621 if (lhs)
|
|
|
622 gimplify_assign (lhs, var, &seq);
|
|
|
623
|
|
|
624 pop_gimplify_context (NULL);
|
|
|
625
|
|
|
626 gsi_replace_with_seq (&gsi, seq, true);
|
|
|
627 }
|
|
|
628
|
|
|
629 /* Implement TARGET_GOACC_REDUCTION.
|
|
|
630
|
|
|
631 Expand calls to the GOACC REDUCTION internal function, into a sequence of
|
|
|
632 gimple instructions. */
|
|
|
633
|
|
|
634 void
|
|
|
635 gcn_goacc_reduction (gcall *call)
|
|
|
636 {
|
|
|
637 int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
|
|
|
638
|
|
|
639 if (level == GOMP_DIM_VECTOR)
|
|
|
640 {
|
|
|
641 default_goacc_reduction (call);
|
|
|
642 return;
|
|
|
643 }
|
|
|
644
|
|
|
645 unsigned code = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 0));
|
|
|
646
|
|
|
647 switch (code)
|
|
|
648 {
|
|
|
649 case IFN_GOACC_REDUCTION_SETUP:
|
|
|
650 gcn_goacc_reduction_setup (call);
|
|
|
651 break;
|
|
|
652
|
|
|
653 case IFN_GOACC_REDUCTION_INIT:
|
|
|
654 gcn_goacc_reduction_init (call);
|
|
|
655 break;
|
|
|
656
|
|
|
657 case IFN_GOACC_REDUCTION_FINI:
|
|
|
658 gcn_goacc_reduction_fini (call);
|
|
|
659 break;
|
|
|
660
|
|
|
661 case IFN_GOACC_REDUCTION_TEARDOWN:
|
|
|
662 gcn_goacc_reduction_teardown (call);
|
|
|
663 break;
|
|
|
664
|
|
|
665 default:
|
|
|
666 gcc_unreachable ();
|
|
|
667 }
|
|
|
668 }
|
|
|
669
|
|
|
670 /* Implement TARGET_GOACC_ADJUST_PROPAGATION_RECORD.
|
|
|
671
|
|
|
672 Tweak (worker) propagation record, e.g. to put it in shared memory. */
|
|
|
673
|
|
|
674 tree
|
|
|
675 gcn_goacc_adjust_propagation_record (tree record_type, bool sender,
|
|
|
676 const char *name)
|
|
|
677 {
|
|
|
678 tree type = record_type;
|
|
|
679
|
|
|
680 TYPE_ADDR_SPACE (type) = ADDR_SPACE_LDS;
|
|
|
681
|
|
|
682 if (!sender)
|
|
|
683 type = build_pointer_type (type);
|
|
|
684
|
|
|
685 tree decl = create_tmp_var_raw (type, name);
|
|
|
686
|
|
|
687 if (sender)
|
|
|
688 {
|
|
|
689 DECL_CONTEXT (decl) = NULL_TREE;
|
|
|
690 TREE_STATIC (decl) = 1;
|
|
|
691 }
|
|
|
692
|
|
|
693 if (sender)
|
|
|
694 varpool_node::finalize_decl (decl);
|
|
|
695
|
|
|
696 return decl;
|
|
|
697 }
|
|
|
698
|
|
|
699 void
|
|
|
700 gcn_goacc_adjust_gangprivate_decl (tree var)
|
|
|
701 {
|
|
|
702 tree type = TREE_TYPE (var);
|
|
|
703 tree lds_type = build_qualified_type (type,
|
|
|
704 TYPE_QUALS_NO_ADDR_SPACE (type)
|
|
|
705 | ENCODE_QUAL_ADDR_SPACE (ADDR_SPACE_LDS));
|
|
|
706 machine_function *machfun = cfun->machine;
|
|
|
707
|
|
|
708 TREE_TYPE (var) = lds_type;
|
|
|
709 TREE_STATIC (var) = 1;
|
|
|
710
|
|
|
711 /* We're making VAR static. We have to mangle the name to avoid collisions
|
|
|
712 between different local variables that share the same names. */
|
|
|
713 lhd_set_decl_assembler_name (var);
|
|
|
714
|
|
|
715 varpool_node::finalize_decl (var);
|
|
|
716
|
|
|
717 if (machfun)
|
|
|
718 machfun->use_flat_addressing = true;
|
|
|
719 }
|
|
|
720
|
|
|
721 /* }}} */
|
