|
111
|
1 /* OMP constructs' SIMD clone supporting code.
|
|
|
2
|
|
|
3 Copyright (C) 2005-2017 Free Software Foundation, Inc.
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
|
21 #include "config.h"
|
|
|
22 #include "system.h"
|
|
|
23 #include "coretypes.h"
|
|
|
24 #include "backend.h"
|
|
|
25 #include "target.h"
|
|
|
26 #include "tree.h"
|
|
|
27 #include "gimple.h"
|
|
|
28 #include "cfghooks.h"
|
|
|
29 #include "alloc-pool.h"
|
|
|
30 #include "tree-pass.h"
|
|
|
31 #include "ssa.h"
|
|
|
32 #include "cgraph.h"
|
|
|
33 #include "pretty-print.h"
|
|
|
34 #include "diagnostic-core.h"
|
|
|
35 #include "fold-const.h"
|
|
|
36 #include "stor-layout.h"
|
|
|
37 #include "cfganal.h"
|
|
|
38 #include "gimplify.h"
|
|
|
39 #include "gimple-iterator.h"
|
|
|
40 #include "gimplify-me.h"
|
|
|
41 #include "gimple-walk.h"
|
|
|
42 #include "langhooks.h"
|
|
|
43 #include "tree-cfg.h"
|
|
|
44 #include "tree-into-ssa.h"
|
|
|
45 #include "tree-dfa.h"
|
|
|
46 #include "cfgloop.h"
|
|
|
47 #include "symbol-summary.h"
|
|
|
48 #include "ipa-prop.h"
|
|
|
49 #include "tree-eh.h"
|
|
|
50 #include "varasm.h"
|
|
|
51 #include "stringpool.h"
|
|
|
52 #include "attribs.h"
|
|
|
53
|
|
|
54 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
|
|
|
55 of arguments to reserve space for. */
|
|
|
56
|
|
|
57 static struct cgraph_simd_clone *
|
|
|
58 simd_clone_struct_alloc (int nargs)
|
|
|
59 {
|
|
|
60 struct cgraph_simd_clone *clone_info;
|
|
|
61 size_t len = (sizeof (struct cgraph_simd_clone)
|
|
|
62 + nargs * sizeof (struct cgraph_simd_clone_arg));
|
|
|
63 clone_info = (struct cgraph_simd_clone *)
|
|
|
64 ggc_internal_cleared_alloc (len);
|
|
|
65 return clone_info;
|
|
|
66 }
|
|
|
67
|
|
|
68 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
|
|
|
69
|
|
|
70 static inline void
|
|
|
71 simd_clone_struct_copy (struct cgraph_simd_clone *to,
|
|
|
72 struct cgraph_simd_clone *from)
|
|
|
73 {
|
|
|
74 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
|
|
|
75 + ((from->nargs - from->inbranch)
|
|
|
76 * sizeof (struct cgraph_simd_clone_arg))));
|
|
|
77 }
|
|
|
78
|
|
|
79 /* Return vector of parameter types of function FNDECL. This uses
|
|
|
80 TYPE_ARG_TYPES if available, otherwise falls back to types of
|
|
|
81 DECL_ARGUMENTS types. */
|
|
|
82
|
|
|
83 static vec<tree>
|
|
|
84 simd_clone_vector_of_formal_parm_types (tree fndecl)
|
|
|
85 {
|
|
|
86 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
|
|
|
87 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
|
|
|
88 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
|
|
|
89 unsigned int i;
|
|
|
90 tree arg;
|
|
|
91 FOR_EACH_VEC_ELT (args, i, arg)
|
|
|
92 args[i] = TREE_TYPE (args[i]);
|
|
|
93 return args;
|
|
|
94 }
|
|
|
95
|
|
|
96 /* Given a simd function in NODE, extract the simd specific
|
|
|
97 information from the OMP clauses passed in CLAUSES, and return
|
|
|
98 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
|
|
|
99 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
|
|
|
100 otherwise set to FALSE. */
|
|
|
101
|
|
|
102 static struct cgraph_simd_clone *
|
|
|
103 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
|
|
|
104 bool *inbranch_specified)
|
|
|
105 {
|
|
|
106 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
|
|
|
107 tree t;
|
|
|
108 int n;
|
|
|
109 *inbranch_specified = false;
|
|
|
110
|
|
|
111 n = args.length ();
|
|
|
112 if (n > 0 && args.last () == void_type_node)
|
|
|
113 n--;
|
|
|
114
|
|
|
115 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
|
|
|
116 be cloned have a distinctive artificial label in addition to "omp
|
|
|
117 declare simd". */
|
|
|
118 bool cilk_clone
|
|
|
119 = (flag_cilkplus
|
|
|
120 && lookup_attribute ("cilk simd function",
|
|
|
121 DECL_ATTRIBUTES (node->decl)));
|
|
|
122
|
|
|
123 /* Allocate one more than needed just in case this is an in-branch
|
|
|
124 clone which will require a mask argument. */
|
|
|
125 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
|
|
|
126 clone_info->nargs = n;
|
|
|
127 clone_info->cilk_elemental = cilk_clone;
|
|
|
128
|
|
|
129 if (!clauses)
|
|
|
130 goto out;
|
|
|
131
|
|
|
132 clauses = TREE_VALUE (clauses);
|
|
|
133 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
|
|
|
134 goto out;
|
|
|
135
|
|
|
136 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
|
|
|
137 {
|
|
|
138 switch (OMP_CLAUSE_CODE (t))
|
|
|
139 {
|
|
|
140 case OMP_CLAUSE_INBRANCH:
|
|
|
141 clone_info->inbranch = 1;
|
|
|
142 *inbranch_specified = true;
|
|
|
143 break;
|
|
|
144 case OMP_CLAUSE_NOTINBRANCH:
|
|
|
145 clone_info->inbranch = 0;
|
|
|
146 *inbranch_specified = true;
|
|
|
147 break;
|
|
|
148 case OMP_CLAUSE_SIMDLEN:
|
|
|
149 clone_info->simdlen
|
|
|
150 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
|
|
|
151 break;
|
|
|
152 case OMP_CLAUSE_LINEAR:
|
|
|
153 {
|
|
|
154 tree decl = OMP_CLAUSE_DECL (t);
|
|
|
155 tree step = OMP_CLAUSE_LINEAR_STEP (t);
|
|
|
156 int argno = TREE_INT_CST_LOW (decl);
|
|
|
157 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
|
|
|
158 {
|
|
|
159 enum cgraph_simd_clone_arg_type arg_type;
|
|
|
160 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
|
|
|
161 switch (OMP_CLAUSE_LINEAR_KIND (t))
|
|
|
162 {
|
|
|
163 case OMP_CLAUSE_LINEAR_REF:
|
|
|
164 arg_type
|
|
|
165 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
|
|
|
166 break;
|
|
|
167 case OMP_CLAUSE_LINEAR_UVAL:
|
|
|
168 arg_type
|
|
|
169 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
|
|
|
170 break;
|
|
|
171 case OMP_CLAUSE_LINEAR_VAL:
|
|
|
172 case OMP_CLAUSE_LINEAR_DEFAULT:
|
|
|
173 arg_type
|
|
|
174 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
|
|
|
175 break;
|
|
|
176 default:
|
|
|
177 gcc_unreachable ();
|
|
|
178 }
|
|
|
179 else
|
|
|
180 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
|
|
|
181 clone_info->args[argno].arg_type = arg_type;
|
|
|
182 clone_info->args[argno].linear_step = tree_to_shwi (step);
|
|
|
183 gcc_assert (clone_info->args[argno].linear_step >= 0
|
|
|
184 && clone_info->args[argno].linear_step < n);
|
|
|
185 }
|
|
|
186 else
|
|
|
187 {
|
|
|
188 if (POINTER_TYPE_P (args[argno]))
|
|
|
189 step = fold_convert (ssizetype, step);
|
|
|
190 if (!tree_fits_shwi_p (step))
|
|
|
191 {
|
|
|
192 warning_at (OMP_CLAUSE_LOCATION (t), 0,
|
|
|
193 "ignoring large linear step");
|
|
|
194 args.release ();
|
|
|
195 return NULL;
|
|
|
196 }
|
|
|
197 else if (integer_zerop (step))
|
|
|
198 {
|
|
|
199 warning_at (OMP_CLAUSE_LOCATION (t), 0,
|
|
|
200 "ignoring zero linear step");
|
|
|
201 args.release ();
|
|
|
202 return NULL;
|
|
|
203 }
|
|
|
204 else
|
|
|
205 {
|
|
|
206 enum cgraph_simd_clone_arg_type arg_type;
|
|
|
207 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
|
|
|
208 switch (OMP_CLAUSE_LINEAR_KIND (t))
|
|
|
209 {
|
|
|
210 case OMP_CLAUSE_LINEAR_REF:
|
|
|
211 arg_type
|
|
|
212 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
|
|
|
213 break;
|
|
|
214 case OMP_CLAUSE_LINEAR_UVAL:
|
|
|
215 arg_type
|
|
|
216 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
|
|
|
217 break;
|
|
|
218 case OMP_CLAUSE_LINEAR_VAL:
|
|
|
219 case OMP_CLAUSE_LINEAR_DEFAULT:
|
|
|
220 arg_type
|
|
|
221 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
|
|
|
222 break;
|
|
|
223 default:
|
|
|
224 gcc_unreachable ();
|
|
|
225 }
|
|
|
226 else
|
|
|
227 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
|
|
|
228 clone_info->args[argno].arg_type = arg_type;
|
|
|
229 clone_info->args[argno].linear_step = tree_to_shwi (step);
|
|
|
230 }
|
|
|
231 }
|
|
|
232 break;
|
|
|
233 }
|
|
|
234 case OMP_CLAUSE_UNIFORM:
|
|
|
235 {
|
|
|
236 tree decl = OMP_CLAUSE_DECL (t);
|
|
|
237 int argno = tree_to_uhwi (decl);
|
|
|
238 clone_info->args[argno].arg_type
|
|
|
239 = SIMD_CLONE_ARG_TYPE_UNIFORM;
|
|
|
240 break;
|
|
|
241 }
|
|
|
242 case OMP_CLAUSE_ALIGNED:
|
|
|
243 {
|
|
|
244 tree decl = OMP_CLAUSE_DECL (t);
|
|
|
245 int argno = tree_to_uhwi (decl);
|
|
|
246 clone_info->args[argno].alignment
|
|
|
247 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
|
|
|
248 break;
|
|
|
249 }
|
|
|
250 default:
|
|
|
251 break;
|
|
|
252 }
|
|
|
253 }
|
|
|
254
|
|
|
255 out:
|
|
|
256 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
|
|
|
257 {
|
|
|
258 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
|
|
|
259 "ignoring %<#pragma omp declare simd%> on function "
|
|
|
260 "with %<_Atomic%> qualified return type");
|
|
|
261 args.release ();
|
|
|
262 return NULL;
|
|
|
263 }
|
|
|
264
|
|
|
265 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
|
|
|
266 if (TYPE_ATOMIC (args[argno])
|
|
|
267 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
|
|
|
268 {
|
|
|
269 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
|
|
|
270 "ignoring %<#pragma omp declare simd%> on function "
|
|
|
271 "with %<_Atomic%> qualified non-%<uniform%> argument");
|
|
|
272 args.release ();
|
|
|
273 return NULL;
|
|
|
274 }
|
|
|
275
|
|
|
276 args.release ();
|
|
|
277 return clone_info;
|
|
|
278 }
|
|
|
279
|
|
|
280 /* Given a SIMD clone in NODE, calculate the characteristic data
|
|
|
281 type and return the coresponding type. The characteristic data
|
|
|
282 type is computed as described in the Intel Vector ABI. */
|
|
|
283
|
|
|
284 static tree
|
|
|
285 simd_clone_compute_base_data_type (struct cgraph_node *node,
|
|
|
286 struct cgraph_simd_clone *clone_info)
|
|
|
287 {
|
|
|
288 tree type = integer_type_node;
|
|
|
289 tree fndecl = node->decl;
|
|
|
290
|
|
|
291 /* a) For non-void function, the characteristic data type is the
|
|
|
292 return type. */
|
|
|
293 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
|
|
|
294 type = TREE_TYPE (TREE_TYPE (fndecl));
|
|
|
295
|
|
|
296 /* b) If the function has any non-uniform, non-linear parameters,
|
|
|
297 then the characteristic data type is the type of the first
|
|
|
298 such parameter. */
|
|
|
299 else
|
|
|
300 {
|
|
|
301 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
|
|
|
302 for (unsigned int i = 0; i < clone_info->nargs; ++i)
|
|
|
303 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
|
|
|
304 {
|
|
|
305 type = map[i];
|
|
|
306 break;
|
|
|
307 }
|
|
|
308 map.release ();
|
|
|
309 }
|
|
|
310
|
|
|
311 /* c) If the characteristic data type determined by a) or b) above
|
|
|
312 is struct, union, or class type which is pass-by-value (except
|
|
|
313 for the type that maps to the built-in complex data type), the
|
|
|
314 characteristic data type is int. */
|
|
|
315 if (RECORD_OR_UNION_TYPE_P (type)
|
|
|
316 && !aggregate_value_p (type, NULL)
|
|
|
317 && TREE_CODE (type) != COMPLEX_TYPE)
|
|
|
318 return integer_type_node;
|
|
|
319
|
|
|
320 /* d) If none of the above three classes is applicable, the
|
|
|
321 characteristic data type is int. */
|
|
|
322
|
|
|
323 return type;
|
|
|
324
|
|
|
325 /* e) For Intel Xeon Phi native and offload compilation, if the
|
|
|
326 resulting characteristic data type is 8-bit or 16-bit integer
|
|
|
327 data type, the characteristic data type is int. */
|
|
|
328 /* Well, we don't handle Xeon Phi yet. */
|
|
|
329 }
|
|
|
330
|
|
|
331 static tree
|
|
|
332 simd_clone_mangle (struct cgraph_node *node,
|
|
|
333 struct cgraph_simd_clone *clone_info)
|
|
|
334 {
|
|
|
335 char vecsize_mangle = clone_info->vecsize_mangle;
|
|
|
336 char mask = clone_info->inbranch ? 'M' : 'N';
|
|
|
337 unsigned int simdlen = clone_info->simdlen;
|
|
|
338 unsigned int n;
|
|
|
339 pretty_printer pp;
|
|
|
340
|
|
|
341 gcc_assert (vecsize_mangle && simdlen);
|
|
|
342
|
|
|
343 pp_string (&pp, "_ZGV");
|
|
|
344 pp_character (&pp, vecsize_mangle);
|
|
|
345 pp_character (&pp, mask);
|
|
|
346 pp_decimal_int (&pp, simdlen);
|
|
|
347
|
|
|
348 for (n = 0; n < clone_info->nargs; ++n)
|
|
|
349 {
|
|
|
350 struct cgraph_simd_clone_arg arg = clone_info->args[n];
|
|
|
351
|
|
|
352 switch (arg.arg_type)
|
|
|
353 {
|
|
|
354 case SIMD_CLONE_ARG_TYPE_UNIFORM:
|
|
|
355 pp_character (&pp, 'u');
|
|
|
356 break;
|
|
|
357 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
|
|
|
358 pp_character (&pp, 'l');
|
|
|
359 goto mangle_linear;
|
|
|
360 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
|
|
|
361 pp_character (&pp, 'R');
|
|
|
362 goto mangle_linear;
|
|
|
363 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
|
|
|
364 pp_character (&pp, 'L');
|
|
|
365 goto mangle_linear;
|
|
|
366 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
|
|
|
367 pp_character (&pp, 'U');
|
|
|
368 goto mangle_linear;
|
|
|
369 mangle_linear:
|
|
|
370 gcc_assert (arg.linear_step != 0);
|
|
|
371 if (arg.linear_step > 1)
|
|
|
372 pp_unsigned_wide_integer (&pp, arg.linear_step);
|
|
|
373 else if (arg.linear_step < 0)
|
|
|
374 {
|
|
|
375 pp_character (&pp, 'n');
|
|
|
376 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
|
|
|
377 arg.linear_step));
|
|
|
378 }
|
|
|
379 break;
|
|
|
380 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
|
|
|
381 pp_string (&pp, "ls");
|
|
|
382 pp_unsigned_wide_integer (&pp, arg.linear_step);
|
|
|
383 break;
|
|
|
384 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
|
|
|
385 pp_string (&pp, "Rs");
|
|
|
386 pp_unsigned_wide_integer (&pp, arg.linear_step);
|
|
|
387 break;
|
|
|
388 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
|
|
|
389 pp_string (&pp, "Ls");
|
|
|
390 pp_unsigned_wide_integer (&pp, arg.linear_step);
|
|
|
391 break;
|
|
|
392 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
|
|
|
393 pp_string (&pp, "Us");
|
|
|
394 pp_unsigned_wide_integer (&pp, arg.linear_step);
|
|
|
395 break;
|
|
|
396 default:
|
|
|
397 pp_character (&pp, 'v');
|
|
|
398 }
|
|
|
399 if (arg.alignment)
|
|
|
400 {
|
|
|
401 pp_character (&pp, 'a');
|
|
|
402 pp_decimal_int (&pp, arg.alignment);
|
|
|
403 }
|
|
|
404 }
|
|
|
405
|
|
|
406 pp_underscore (&pp);
|
|
|
407 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
|
|
|
408 if (*str == '*')
|
|
|
409 ++str;
|
|
|
410 pp_string (&pp, str);
|
|
|
411 str = pp_formatted_text (&pp);
|
|
|
412
|
|
|
413 /* If there already is a SIMD clone with the same mangled name, don't
|
|
|
414 add another one. This can happen e.g. for
|
|
|
415 #pragma omp declare simd
|
|
|
416 #pragma omp declare simd simdlen(8)
|
|
|
417 int foo (int, int);
|
|
|
418 if the simdlen is assumed to be 8 for the first one, etc. */
|
|
|
419 for (struct cgraph_node *clone = node->simd_clones; clone;
|
|
|
420 clone = clone->simdclone->next_clone)
|
|
|
421 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
|
|
|
422 return NULL_TREE;
|
|
|
423
|
|
|
424 return get_identifier (str);
|
|
|
425 }
|
|
|
426
|
|
|
427 /* Create a simd clone of OLD_NODE and return it. */
|
|
|
428
|
|
|
429 static struct cgraph_node *
|
|
|
430 simd_clone_create (struct cgraph_node *old_node)
|
|
|
431 {
|
|
|
432 struct cgraph_node *new_node;
|
|
|
433 if (old_node->definition)
|
|
|
434 {
|
|
|
435 if (!old_node->has_gimple_body_p ())
|
|
|
436 return NULL;
|
|
|
437 old_node->get_body ();
|
|
|
438 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
|
|
|
439 false, NULL, NULL,
|
|
|
440 "simdclone");
|
|
|
441 }
|
|
|
442 else
|
|
|
443 {
|
|
|
444 tree old_decl = old_node->decl;
|
|
|
445 tree new_decl = copy_node (old_node->decl);
|
|
|
446 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
|
|
|
447 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
|
|
|
448 SET_DECL_RTL (new_decl, NULL);
|
|
|
449 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
|
|
|
450 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
|
|
|
451 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
|
|
|
452 if (old_node->in_other_partition)
|
|
|
453 new_node->in_other_partition = 1;
|
|
|
454 }
|
|
|
455 if (new_node == NULL)
|
|
|
456 return new_node;
|
|
|
457
|
|
|
458 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
|
|
|
459 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
|
|
|
460 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
|
|
|
461 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
|
|
|
462 DECL_VISIBILITY_SPECIFIED (new_node->decl)
|
|
|
463 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
|
|
|
464 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
|
|
|
465 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
|
|
|
466 if (DECL_ONE_ONLY (old_node->decl))
|
|
|
467 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
|
|
|
468
|
|
|
469 /* The method cgraph_version_clone_with_body () will force the new
|
|
|
470 symbol local. Undo this, and inherit external visibility from
|
|
|
471 the old node. */
|
|
|
472 new_node->local.local = old_node->local.local;
|
|
|
473 new_node->externally_visible = old_node->externally_visible;
|
|
|
474
|
|
|
475 return new_node;
|
|
|
476 }
|
|
|
477
|
|
|
478 /* Adjust the return type of the given function to its appropriate
|
|
|
479 vector counterpart. Returns a simd array to be used throughout the
|
|
|
480 function as a return value. */
|
|
|
481
|
|
|
482 static tree
|
|
|
483 simd_clone_adjust_return_type (struct cgraph_node *node)
|
|
|
484 {
|
|
|
485 tree fndecl = node->decl;
|
|
|
486 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
|
|
|
487 unsigned int veclen;
|
|
|
488 tree t;
|
|
|
489
|
|
|
490 /* Adjust the function return type. */
|
|
|
491 if (orig_rettype == void_type_node)
|
|
|
492 return NULL_TREE;
|
|
|
493 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
|
|
|
494 t = TREE_TYPE (TREE_TYPE (fndecl));
|
|
|
495 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
|
|
|
496 veclen = node->simdclone->vecsize_int;
|
|
|
497 else
|
|
|
498 veclen = node->simdclone->vecsize_float;
|
|
|
499 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
|
|
|
500 if (veclen > node->simdclone->simdlen)
|
|
|
501 veclen = node->simdclone->simdlen;
|
|
|
502 if (POINTER_TYPE_P (t))
|
|
|
503 t = pointer_sized_int_node;
|
|
|
504 if (veclen == node->simdclone->simdlen)
|
|
|
505 t = build_vector_type (t, node->simdclone->simdlen);
|
|
|
506 else
|
|
|
507 {
|
|
|
508 t = build_vector_type (t, veclen);
|
|
|
509 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
|
|
|
510 }
|
|
|
511 TREE_TYPE (TREE_TYPE (fndecl)) = t;
|
|
|
512 if (!node->definition)
|
|
|
513 return NULL_TREE;
|
|
|
514
|
|
|
515 t = DECL_RESULT (fndecl);
|
|
|
516 /* Adjust the DECL_RESULT. */
|
|
|
517 gcc_assert (TREE_TYPE (t) != void_type_node);
|
|
|
518 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
|
|
|
519 relayout_decl (t);
|
|
|
520
|
|
|
521 tree atype = build_array_type_nelts (orig_rettype,
|
|
|
522 node->simdclone->simdlen);
|
|
|
523 if (veclen != node->simdclone->simdlen)
|
|
|
524 return build1 (VIEW_CONVERT_EXPR, atype, t);
|
|
|
525
|
|
|
526 /* Set up a SIMD array to use as the return value. */
|
|
|
527 tree retval = create_tmp_var_raw (atype, "retval");
|
|
|
528 gimple_add_tmp_var (retval);
|
|
|
529 return retval;
|
|
|
530 }
|
|
|
531
|
|
|
532 /* Each vector argument has a corresponding array to be used locally
|
|
|
533 as part of the eventual loop. Create such temporary array and
|
|
|
534 return it.
|
|
|
535
|
|
|
536 PREFIX is the prefix to be used for the temporary.
|
|
|
537
|
|
|
538 TYPE is the inner element type.
|
|
|
539
|
|
|
540 SIMDLEN is the number of elements. */
|
|
|
541
|
|
|
542 static tree
|
|
|
543 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
|
|
|
544 {
|
|
|
545 tree atype = build_array_type_nelts (type, simdlen);
|
|
|
546 tree avar = create_tmp_var_raw (atype, prefix);
|
|
|
547 gimple_add_tmp_var (avar);
|
|
|
548 return avar;
|
|
|
549 }
|
|
|
550
|
|
|
551 /* Modify the function argument types to their corresponding vector
|
|
|
552 counterparts if appropriate. Also, create one array for each simd
|
|
|
553 argument to be used locally when using the function arguments as
|
|
|
554 part of the loop.
|
|
|
555
|
|
|
556 NODE is the function whose arguments are to be adjusted.
|
|
|
557
|
|
|
558 Returns an adjustment vector that will be filled describing how the
|
|
|
559 argument types will be adjusted. */
|
|
|
560
|
|
|
561 static ipa_parm_adjustment_vec
|
|
|
562 simd_clone_adjust_argument_types (struct cgraph_node *node)
|
|
|
563 {
|
|
|
564 vec<tree> args;
|
|
|
565 ipa_parm_adjustment_vec adjustments;
|
|
|
566
|
|
|
567 if (node->definition)
|
|
|
568 args = ipa_get_vector_of_formal_parms (node->decl);
|
|
|
569 else
|
|
|
570 args = simd_clone_vector_of_formal_parm_types (node->decl);
|
|
|
571 adjustments.create (args.length ());
|
|
|
572 unsigned i, j, veclen;
|
|
|
573 struct ipa_parm_adjustment adj;
|
|
|
574 struct cgraph_simd_clone *sc = node->simdclone;
|
|
|
575
|
|
|
576 for (i = 0; i < sc->nargs; ++i)
|
|
|
577 {
|
|
|
578 memset (&adj, 0, sizeof (adj));
|
|
|
579 tree parm = args[i];
|
|
|
580 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
|
|
|
581 adj.base_index = i;
|
|
|
582 adj.base = parm;
|
|
|
583
|
|
|
584 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
|
|
|
585 sc->args[i].orig_type = parm_type;
|
|
|
586
|
|
|
587 switch (sc->args[i].arg_type)
|
|
|
588 {
|
|
|
589 default:
|
|
|
590 /* No adjustment necessary for scalar arguments. */
|
|
|
591 adj.op = IPA_PARM_OP_COPY;
|
|
|
592 break;
|
|
|
593 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
|
|
|
594 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
|
|
|
595 if (node->definition)
|
|
|
596 sc->args[i].simd_array
|
|
|
597 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
|
|
|
598 TREE_TYPE (parm_type),
|
|
|
599 sc->simdlen);
|
|
|
600 adj.op = IPA_PARM_OP_COPY;
|
|
|
601 break;
|
|
|
602 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
|
|
|
603 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
|
|
|
604 case SIMD_CLONE_ARG_TYPE_VECTOR:
|
|
|
605 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
|
|
|
606 veclen = sc->vecsize_int;
|
|
|
607 else
|
|
|
608 veclen = sc->vecsize_float;
|
|
|
609 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
|
|
|
610 if (veclen > sc->simdlen)
|
|
|
611 veclen = sc->simdlen;
|
|
|
612 adj.arg_prefix = "simd";
|
|
|
613 if (POINTER_TYPE_P (parm_type))
|
|
|
614 adj.type = build_vector_type (pointer_sized_int_node, veclen);
|
|
|
615 else
|
|
|
616 adj.type = build_vector_type (parm_type, veclen);
|
|
|
617 sc->args[i].vector_type = adj.type;
|
|
|
618 for (j = veclen; j < sc->simdlen; j += veclen)
|
|
|
619 {
|
|
|
620 adjustments.safe_push (adj);
|
|
|
621 if (j == veclen)
|
|
|
622 {
|
|
|
623 memset (&adj, 0, sizeof (adj));
|
|
|
624 adj.op = IPA_PARM_OP_NEW;
|
|
|
625 adj.arg_prefix = "simd";
|
|
|
626 adj.base_index = i;
|
|
|
627 adj.type = sc->args[i].vector_type;
|
|
|
628 }
|
|
|
629 }
|
|
|
630
|
|
|
631 if (node->definition)
|
|
|
632 sc->args[i].simd_array
|
|
|
633 = create_tmp_simd_array (DECL_NAME (parm)
|
|
|
634 ? IDENTIFIER_POINTER (DECL_NAME (parm))
|
|
|
635 : NULL, parm_type, sc->simdlen);
|
|
|
636 }
|
|
|
637 adjustments.safe_push (adj);
|
|
|
638 }
|
|
|
639
|
|
|
640 if (sc->inbranch)
|
|
|
641 {
|
|
|
642 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
|
|
|
643
|
|
|
644 memset (&adj, 0, sizeof (adj));
|
|
|
645 adj.op = IPA_PARM_OP_NEW;
|
|
|
646 adj.arg_prefix = "mask";
|
|
|
647
|
|
|
648 adj.base_index = i;
|
|
|
649 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
|
|
|
650 veclen = sc->vecsize_int;
|
|
|
651 else
|
|
|
652 veclen = sc->vecsize_float;
|
|
|
653 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
|
|
|
654 if (veclen > sc->simdlen)
|
|
|
655 veclen = sc->simdlen;
|
|
|
656 if (sc->mask_mode != VOIDmode)
|
|
|
657 adj.type
|
|
|
658 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
|
|
|
659 else if (POINTER_TYPE_P (base_type))
|
|
|
660 adj.type = build_vector_type (pointer_sized_int_node, veclen);
|
|
|
661 else
|
|
|
662 adj.type = build_vector_type (base_type, veclen);
|
|
|
663 adjustments.safe_push (adj);
|
|
|
664
|
|
|
665 for (j = veclen; j < sc->simdlen; j += veclen)
|
|
|
666 adjustments.safe_push (adj);
|
|
|
667
|
|
|
668 /* We have previously allocated one extra entry for the mask. Use
|
|
|
669 it and fill it. */
|
|
|
670 sc->nargs++;
|
|
|
671 if (sc->mask_mode != VOIDmode)
|
|
|
672 base_type = boolean_type_node;
|
|
|
673 if (node->definition)
|
|
|
674 {
|
|
|
675 sc->args[i].orig_arg
|
|
|
676 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
|
|
|
677 if (sc->mask_mode == VOIDmode)
|
|
|
678 sc->args[i].simd_array
|
|
|
679 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
|
|
|
680 else if (veclen < sc->simdlen)
|
|
|
681 sc->args[i].simd_array
|
|
|
682 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
|
|
|
683 else
|
|
|
684 sc->args[i].simd_array = NULL_TREE;
|
|
|
685 }
|
|
|
686 sc->args[i].orig_type = base_type;
|
|
|
687 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
|
|
|
688 }
|
|
|
689
|
|
|
690 if (node->definition)
|
|
|
691 ipa_modify_formal_parameters (node->decl, adjustments);
|
|
|
692 else
|
|
|
693 {
|
|
|
694 tree new_arg_types = NULL_TREE, new_reversed;
|
|
|
695 bool last_parm_void = false;
|
|
|
696 if (args.length () > 0 && args.last () == void_type_node)
|
|
|
697 last_parm_void = true;
|
|
|
698
|
|
|
699 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
|
|
|
700 j = adjustments.length ();
|
|
|
701 for (i = 0; i < j; i++)
|
|
|
702 {
|
|
|
703 struct ipa_parm_adjustment *adj = &adjustments[i];
|
|
|
704 tree ptype;
|
|
|
705 if (adj->op == IPA_PARM_OP_COPY)
|
|
|
706 ptype = args[adj->base_index];
|
|
|
707 else
|
|
|
708 ptype = adj->type;
|
|
|
709 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
|
|
|
710 }
|
|
|
711 new_reversed = nreverse (new_arg_types);
|
|
|
712 if (last_parm_void)
|
|
|
713 {
|
|
|
714 if (new_reversed)
|
|
|
715 TREE_CHAIN (new_arg_types) = void_list_node;
|
|
|
716 else
|
|
|
717 new_reversed = void_list_node;
|
|
|
718 }
|
|
|
719
|
|
|
720 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
|
|
|
721 TYPE_ARG_TYPES (new_type) = new_reversed;
|
|
|
722 TREE_TYPE (node->decl) = new_type;
|
|
|
723
|
|
|
724 adjustments.release ();
|
|
|
725 }
|
|
|
726 args.release ();
|
|
|
727 return adjustments;
|
|
|
728 }
|
|
|
729
|
|
|
730 /* Initialize and copy the function arguments in NODE to their
|
|
|
731 corresponding local simd arrays. Returns a fresh gimple_seq with
|
|
|
732 the instruction sequence generated. */
|
|
|
733
|
|
|
734 static gimple_seq
|
|
|
735 simd_clone_init_simd_arrays (struct cgraph_node *node,
|
|
|
736 ipa_parm_adjustment_vec adjustments)
|
|
|
737 {
|
|
|
738 gimple_seq seq = NULL;
|
|
|
739 unsigned i = 0, j = 0, k;
|
|
|
740
|
|
|
741 for (tree arg = DECL_ARGUMENTS (node->decl);
|
|
|
742 arg;
|
|
|
743 arg = DECL_CHAIN (arg), i++, j++)
|
|
|
744 {
|
|
|
745 if (adjustments[j].op == IPA_PARM_OP_COPY
|
|
|
746 || POINTER_TYPE_P (TREE_TYPE (arg)))
|
|
|
747 continue;
|
|
|
748
|
|
|
749 node->simdclone->args[i].vector_arg = arg;
|
|
|
750
|
|
|
751 tree array = node->simdclone->args[i].simd_array;
|
|
|
752 if (node->simdclone->mask_mode != VOIDmode
|
|
|
753 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
|
|
|
754 {
|
|
|
755 if (array == NULL_TREE)
|
|
|
756 continue;
|
|
|
757 unsigned int l
|
|
|
758 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
|
|
|
759 for (k = 0; k <= l; k++)
|
|
|
760 {
|
|
|
761 if (k)
|
|
|
762 {
|
|
|
763 arg = DECL_CHAIN (arg);
|
|
|
764 j++;
|
|
|
765 }
|
|
|
766 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
|
|
|
767 array, size_int (k), NULL, NULL);
|
|
|
768 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
|
|
|
769 gimplify_and_add (t, &seq);
|
|
|
770 }
|
|
|
771 continue;
|
|
|
772 }
|
|
|
773 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
|
|
|
774 {
|
|
|
775 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
|
|
|
776 tree ptr = build_fold_addr_expr (array);
|
|
|
777 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
|
|
|
778 build_int_cst (ptype, 0));
|
|
|
779 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
|
|
|
780 gimplify_and_add (t, &seq);
|
|
|
781 }
|
|
|
782 else
|
|
|
783 {
|
|
|
784 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
|
|
|
785 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
|
|
|
786 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
|
|
|
787 {
|
|
|
788 tree ptr = build_fold_addr_expr (array);
|
|
|
789 int elemsize;
|
|
|
790 if (k)
|
|
|
791 {
|
|
|
792 arg = DECL_CHAIN (arg);
|
|
|
793 j++;
|
|
|
794 }
|
|
|
795 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
|
|
|
796 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
|
|
|
797 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
|
|
|
798 build_int_cst (ptype, k * elemsize));
|
|
|
799 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
|
|
|
800 gimplify_and_add (t, &seq);
|
|
|
801 }
|
|
|
802 }
|
|
|
803 }
|
|
|
804 return seq;
|
|
|
805 }
|
|
|
806
|
|
|
807 /* Callback info for ipa_simd_modify_stmt_ops below. */
|
|
|
808
|
|
|
809 struct modify_stmt_info {
|
|
|
810 ipa_parm_adjustment_vec adjustments;
|
|
|
811 gimple *stmt;
|
|
|
812 /* True if the parent statement was modified by
|
|
|
813 ipa_simd_modify_stmt_ops. */
|
|
|
814 bool modified;
|
|
|
815 };
|
|
|
816
|
|
|
817 /* Callback for walk_gimple_op.
|
|
|
818
|
|
|
819 Adjust operands from a given statement as specified in the
|
|
|
820 adjustments vector in the callback data. */
|
|
|
821
|
|
|
822 static tree
|
|
|
823 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
|
|
|
824 {
|
|
|
825 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
|
|
|
826 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
|
|
|
827 tree *orig_tp = tp;
|
|
|
828 if (TREE_CODE (*tp) == ADDR_EXPR)
|
|
|
829 tp = &TREE_OPERAND (*tp, 0);
|
|
|
830 struct ipa_parm_adjustment *cand = NULL;
|
|
|
831 if (TREE_CODE (*tp) == PARM_DECL)
|
|
|
832 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
|
|
|
833 else
|
|
|
834 {
|
|
|
835 if (TYPE_P (*tp))
|
|
|
836 *walk_subtrees = 0;
|
|
|
837 }
|
|
|
838
|
|
|
839 tree repl = NULL_TREE;
|
|
|
840 if (cand)
|
|
|
841 repl = unshare_expr (cand->new_decl);
|
|
|
842 else
|
|
|
843 {
|
|
|
844 if (tp != orig_tp)
|
|
|
845 {
|
|
|
846 *walk_subtrees = 0;
|
|
|
847 bool modified = info->modified;
|
|
|
848 info->modified = false;
|
|
|
849 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
|
|
|
850 if (!info->modified)
|
|
|
851 {
|
|
|
852 info->modified = modified;
|
|
|
853 return NULL_TREE;
|
|
|
854 }
|
|
|
855 info->modified = modified;
|
|
|
856 repl = *tp;
|
|
|
857 }
|
|
|
858 else
|
|
|
859 return NULL_TREE;
|
|
|
860 }
|
|
|
861
|
|
|
862 if (tp != orig_tp)
|
|
|
863 {
|
|
|
864 repl = build_fold_addr_expr (repl);
|
|
|
865 gimple *stmt;
|
|
|
866 if (is_gimple_debug (info->stmt))
|
|
|
867 {
|
|
|
868 tree vexpr = make_node (DEBUG_EXPR_DECL);
|
|
|
869 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
|
|
|
870 DECL_ARTIFICIAL (vexpr) = 1;
|
|
|
871 TREE_TYPE (vexpr) = TREE_TYPE (repl);
|
|
|
872 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
|
|
|
873 repl = vexpr;
|
|
|
874 }
|
|
|
875 else
|
|
|
876 {
|
|
|
877 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
|
|
|
878 repl = gimple_assign_lhs (stmt);
|
|
|
879 }
|
|
|
880 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
|
|
|
881 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
|
|
|
882 *orig_tp = repl;
|
|
|
883 }
|
|
|
884 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
|
|
|
885 {
|
|
|
886 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
|
|
|
887 *tp = vce;
|
|
|
888 }
|
|
|
889 else
|
|
|
890 *tp = repl;
|
|
|
891
|
|
|
892 info->modified = true;
|
|
|
893 return NULL_TREE;
|
|
|
894 }
|
|
|
895
|
|
|
896 /* Traverse the function body and perform all modifications as
|
|
|
897 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
|
|
|
898 modified such that the replacement/reduction value will now be an
|
|
|
899 offset into the corresponding simd_array.
|
|
|
900
|
|
|
901 This function will replace all function argument uses with their
|
|
|
902 corresponding simd array elements, and ajust the return values
|
|
|
903 accordingly. */
|
|
|
904
|
|
|
905 static void
|
|
|
906 ipa_simd_modify_function_body (struct cgraph_node *node,
|
|
|
907 ipa_parm_adjustment_vec adjustments,
|
|
|
908 tree retval_array, tree iter)
|
|
|
909 {
|
|
|
910 basic_block bb;
|
|
|
911 unsigned int i, j, l;
|
|
|
912
|
|
|
913 /* Re-use the adjustments array, but this time use it to replace
|
|
|
914 every function argument use to an offset into the corresponding
|
|
|
915 simd_array. */
|
|
|
916 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
|
|
|
917 {
|
|
|
918 if (!node->simdclone->args[i].vector_arg)
|
|
|
919 continue;
|
|
|
920
|
|
|
921 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
|
|
|
922 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
|
|
|
923 adjustments[j].new_decl
|
|
|
924 = build4 (ARRAY_REF,
|
|
|
925 basetype,
|
|
|
926 node->simdclone->args[i].simd_array,
|
|
|
927 iter,
|
|
|
928 NULL_TREE, NULL_TREE);
|
|
|
929 if (adjustments[j].op == IPA_PARM_OP_NONE
|
|
|
930 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
|
|
|
931 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
|
|
|
932 }
|
|
|
933
|
|
|
934 l = adjustments.length ();
|
|
|
935 tree name;
|
|
|
936
|
|
|
937 FOR_EACH_SSA_NAME (i, name, cfun)
|
|
|
938 {
|
|
|
939 if (SSA_NAME_VAR (name)
|
|
|
940 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
|
|
|
941 {
|
|
|
942 for (j = 0; j < l; j++)
|
|
|
943 if (SSA_NAME_VAR (name) == adjustments[j].base
|
|
|
944 && adjustments[j].new_decl)
|
|
|
945 {
|
|
|
946 tree base_var;
|
|
|
947 if (adjustments[j].new_ssa_base == NULL_TREE)
|
|
|
948 {
|
|
|
949 base_var
|
|
|
950 = copy_var_decl (adjustments[j].base,
|
|
|
951 DECL_NAME (adjustments[j].base),
|
|
|
952 TREE_TYPE (adjustments[j].base));
|
|
|
953 adjustments[j].new_ssa_base = base_var;
|
|
|
954 }
|
|
|
955 else
|
|
|
956 base_var = adjustments[j].new_ssa_base;
|
|
|
957 if (SSA_NAME_IS_DEFAULT_DEF (name))
|
|
|
958 {
|
|
|
959 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
|
|
|
960 gimple_stmt_iterator gsi = gsi_after_labels (bb);
|
|
|
961 tree new_decl = unshare_expr (adjustments[j].new_decl);
|
|
|
962 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
|
|
|
963 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
|
|
|
964 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
|
|
|
965 gimple *stmt = gimple_build_assign (name, new_decl);
|
|
|
966 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
|
|
|
967 }
|
|
|
968 else
|
|
|
969 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
|
|
|
970 }
|
|
|
971 }
|
|
|
972 }
|
|
|
973
|
|
|
974 struct modify_stmt_info info;
|
|
|
975 info.adjustments = adjustments;
|
|
|
976
|
|
|
977 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
|
|
|
978 {
|
|
|
979 gimple_stmt_iterator gsi;
|
|
|
980
|
|
|
981 gsi = gsi_start_bb (bb);
|
|
|
982 while (!gsi_end_p (gsi))
|
|
|
983 {
|
|
|
984 gimple *stmt = gsi_stmt (gsi);
|
|
|
985 info.stmt = stmt;
|
|
|
986 struct walk_stmt_info wi;
|
|
|
987
|
|
|
988 memset (&wi, 0, sizeof (wi));
|
|
|
989 info.modified = false;
|
|
|
990 wi.info = &info;
|
|
|
991 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
|
|
|
992
|
|
|
993 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
|
|
|
994 {
|
|
|
995 tree retval = gimple_return_retval (return_stmt);
|
|
|
996 if (!retval)
|
|
|
997 {
|
|
|
998 gsi_remove (&gsi, true);
|
|
|
999 continue;
|
|
|
1000 }
|
|
|
1001
|
|
|
1002 /* Replace `return foo' with `retval_array[iter] = foo'. */
|
|
|
1003 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
|
|
|
1004 retval_array, iter, NULL, NULL);
|
|
|
1005 stmt = gimple_build_assign (ref, retval);
|
|
|
1006 gsi_replace (&gsi, stmt, true);
|
|
|
1007 info.modified = true;
|
|
|
1008 }
|
|
|
1009
|
|
|
1010 if (info.modified)
|
|
|
1011 {
|
|
|
1012 update_stmt (stmt);
|
|
|
1013 if (maybe_clean_eh_stmt (stmt))
|
|
|
1014 gimple_purge_dead_eh_edges (gimple_bb (stmt));
|
|
|
1015 }
|
|
|
1016 gsi_next (&gsi);
|
|
|
1017 }
|
|
|
1018 }
|
|
|
1019 }
|
|
|
1020
|
|
|
1021 /* Helper function of simd_clone_adjust, return linear step addend
|
|
|
1022 of Ith argument. */
|
|
|
1023
|
|
|
1024 static tree
|
|
|
1025 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
|
|
|
1026 tree addtype, basic_block entry_bb)
|
|
|
1027 {
|
|
|
1028 tree ptype = NULL_TREE;
|
|
|
1029 switch (node->simdclone->args[i].arg_type)
|
|
|
1030 {
|
|
|
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
|
|
|
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
|
|
|
1033 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
|
|
|
1034 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
|
|
|
1035 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
|
|
|
1036 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
|
|
|
1037 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
|
|
|
1038 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
|
|
|
1039 break;
|
|
|
1040 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
|
|
|
1041 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
|
|
|
1042 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
|
|
|
1043 break;
|
|
|
1044 default:
|
|
|
1045 gcc_unreachable ();
|
|
|
1046 }
|
|
|
1047
|
|
|
1048 unsigned int idx = node->simdclone->args[i].linear_step;
|
|
|
1049 tree arg = node->simdclone->args[idx].orig_arg;
|
|
|
1050 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
|
|
|
1051 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
|
|
|
1052 gimple *g;
|
|
|
1053 tree ret;
|
|
|
1054 if (is_gimple_reg (arg))
|
|
|
1055 ret = get_or_create_ssa_default_def (cfun, arg);
|
|
|
1056 else
|
|
|
1057 {
|
|
|
1058 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
|
|
|
1059 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1060 ret = gimple_assign_lhs (g);
|
|
|
1061 }
|
|
|
1062 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
|
|
|
1063 {
|
|
|
1064 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
|
|
|
1065 build_simple_mem_ref (ret));
|
|
|
1066 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1067 ret = gimple_assign_lhs (g);
|
|
|
1068 }
|
|
|
1069 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
|
|
|
1070 {
|
|
|
1071 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
|
|
|
1072 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1073 ret = gimple_assign_lhs (g);
|
|
|
1074 }
|
|
|
1075 if (POINTER_TYPE_P (ptype))
|
|
|
1076 {
|
|
|
1077 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
|
|
|
1078 if (size && TREE_CODE (size) == INTEGER_CST)
|
|
|
1079 {
|
|
|
1080 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
|
|
|
1081 ret, fold_convert (addtype, size));
|
|
|
1082 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1083 ret = gimple_assign_lhs (g);
|
|
|
1084 }
|
|
|
1085 }
|
|
|
1086 return ret;
|
|
|
1087 }
|
|
|
1088
|
|
|
1089 /* Adjust the argument types in NODE to their appropriate vector
|
|
|
1090 counterparts. */
|
|
|
1091
|
|
|
1092 static void
|
|
|
1093 simd_clone_adjust (struct cgraph_node *node)
|
|
|
1094 {
|
|
|
1095 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
|
|
|
1096
|
|
|
1097 targetm.simd_clone.adjust (node);
|
|
|
1098
|
|
|
1099 tree retval = simd_clone_adjust_return_type (node);
|
|
|
1100 ipa_parm_adjustment_vec adjustments
|
|
|
1101 = simd_clone_adjust_argument_types (node);
|
|
|
1102
|
|
|
1103 push_gimplify_context ();
|
|
|
1104
|
|
|
1105 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
|
|
|
1106
|
|
|
1107 /* Adjust all uses of vector arguments accordingly. Adjust all
|
|
|
1108 return values accordingly. */
|
|
|
1109 tree iter = create_tmp_var (unsigned_type_node, "iter");
|
|
|
1110 tree iter1 = make_ssa_name (iter);
|
|
|
1111 tree iter2 = NULL_TREE;
|
|
|
1112 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
|
|
|
1113 adjustments.release ();
|
|
|
1114
|
|
|
1115 /* Initialize the iteration variable. */
|
|
|
1116 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
|
|
|
1117 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
|
|
|
1118 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
|
|
|
1119 /* Insert the SIMD array and iv initialization at function
|
|
|
1120 entry. */
|
|
|
1121 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
|
|
|
1122
|
|
|
1123 pop_gimplify_context (NULL);
|
|
|
1124
|
|
|
1125 gimple *g;
|
|
|
1126 basic_block incr_bb = NULL;
|
|
|
1127 struct loop *loop = NULL;
|
|
|
1128
|
|
|
1129 /* Create a new BB right before the original exit BB, to hold the
|
|
|
1130 iteration increment and the condition/branch. */
|
|
|
1131 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
|
|
|
1132 {
|
|
|
1133 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
|
|
|
1134 incr_bb = create_empty_bb (orig_exit);
|
|
|
1135 add_bb_to_loop (incr_bb, body_bb->loop_father);
|
|
|
1136 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
|
|
|
1137 flag. Set it now to be a FALLTHRU_EDGE. */
|
|
|
1138 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
|
|
|
1139 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
|
|
|
1140 for (unsigned i = 0;
|
|
|
1141 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
|
|
|
1142 {
|
|
|
1143 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
|
|
|
1144 redirect_edge_succ (e, incr_bb);
|
|
|
1145 }
|
|
|
1146 }
|
|
|
1147 else if (node->simdclone->inbranch)
|
|
|
1148 {
|
|
|
1149 incr_bb = create_empty_bb (entry_bb);
|
|
|
1150 add_bb_to_loop (incr_bb, body_bb->loop_father);
|
|
|
1151 }
|
|
|
1152
|
|
|
1153 if (incr_bb)
|
|
|
1154 {
|
|
|
1155 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
|
|
|
1156 gsi = gsi_last_bb (incr_bb);
|
|
|
1157 iter2 = make_ssa_name (iter);
|
|
|
1158 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
|
|
|
1159 build_int_cst (unsigned_type_node, 1));
|
|
|
1160 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1161
|
|
|
1162 /* Mostly annotate the loop for the vectorizer (the rest is done
|
|
|
1163 below). */
|
|
|
1164 loop = alloc_loop ();
|
|
|
1165 cfun->has_force_vectorize_loops = true;
|
|
|
1166 loop->safelen = node->simdclone->simdlen;
|
|
|
1167 loop->force_vectorize = true;
|
|
|
1168 loop->header = body_bb;
|
|
|
1169 }
|
|
|
1170
|
|
|
1171 /* Branch around the body if the mask applies. */
|
|
|
1172 if (node->simdclone->inbranch)
|
|
|
1173 {
|
|
|
1174 gsi = gsi_last_bb (loop->header);
|
|
|
1175 tree mask_array
|
|
|
1176 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
|
|
|
1177 tree mask;
|
|
|
1178 if (node->simdclone->mask_mode != VOIDmode)
|
|
|
1179 {
|
|
|
1180 tree shift_cnt;
|
|
|
1181 if (mask_array == NULL_TREE)
|
|
|
1182 {
|
|
|
1183 tree arg = node->simdclone->args[node->simdclone->nargs
|
|
|
1184 - 1].vector_arg;
|
|
|
1185 mask = get_or_create_ssa_default_def (cfun, arg);
|
|
|
1186 shift_cnt = iter1;
|
|
|
1187 }
|
|
|
1188 else
|
|
|
1189 {
|
|
|
1190 tree maskt = TREE_TYPE (mask_array);
|
|
|
1191 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
|
|
|
1192 c = node->simdclone->simdlen / (c + 1);
|
|
|
1193 int s = exact_log2 (c);
|
|
|
1194 gcc_assert (s > 0);
|
|
|
1195 c--;
|
|
|
1196 tree idx = make_ssa_name (TREE_TYPE (iter1));
|
|
|
1197 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
|
|
|
1198 build_int_cst (NULL_TREE, s));
|
|
|
1199 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1200 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
|
|
|
1201 tree aref = build4 (ARRAY_REF,
|
|
|
1202 TREE_TYPE (TREE_TYPE (mask_array)),
|
|
|
1203 mask_array, idx, NULL, NULL);
|
|
|
1204 g = gimple_build_assign (mask, aref);
|
|
|
1205 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1206 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
|
|
|
1207 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
|
|
|
1208 build_int_cst (TREE_TYPE (iter1), c));
|
|
|
1209 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1210 }
|
|
|
1211 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
|
|
|
1212 RSHIFT_EXPR, mask, shift_cnt);
|
|
|
1213 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1214 mask = gimple_assign_lhs (g);
|
|
|
1215 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
|
|
|
1216 BIT_AND_EXPR, mask,
|
|
|
1217 build_int_cst (TREE_TYPE (mask), 1));
|
|
|
1218 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1219 mask = gimple_assign_lhs (g);
|
|
|
1220 }
|
|
|
1221 else
|
|
|
1222 {
|
|
|
1223 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
|
|
|
1224 tree aref = build4 (ARRAY_REF,
|
|
|
1225 TREE_TYPE (TREE_TYPE (mask_array)),
|
|
|
1226 mask_array, iter1, NULL, NULL);
|
|
|
1227 g = gimple_build_assign (mask, aref);
|
|
|
1228 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1229 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
|
|
|
1230 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
|
|
|
1231 {
|
|
|
1232 aref = build1 (VIEW_CONVERT_EXPR,
|
|
|
1233 build_nonstandard_integer_type (bitsize, 0),
|
|
|
1234 mask);
|
|
|
1235 mask = make_ssa_name (TREE_TYPE (aref));
|
|
|
1236 g = gimple_build_assign (mask, aref);
|
|
|
1237 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1238 }
|
|
|
1239 }
|
|
|
1240
|
|
|
1241 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
|
|
|
1242 NULL, NULL);
|
|
|
1243 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1244 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
|
|
|
1245 e->probability = profile_probability::unlikely ().guessed ();
|
|
|
1246 edge fallthru = FALLTHRU_EDGE (loop->header);
|
|
|
1247 fallthru->flags = EDGE_FALSE_VALUE;
|
|
|
1248 fallthru->probability = profile_probability::likely ().guessed ();
|
|
|
1249 }
|
|
|
1250
|
|
|
1251 basic_block latch_bb = NULL;
|
|
|
1252 basic_block new_exit_bb = NULL;
|
|
|
1253
|
|
|
1254 /* Generate the condition. */
|
|
|
1255 if (incr_bb)
|
|
|
1256 {
|
|
|
1257 gsi = gsi_last_bb (incr_bb);
|
|
|
1258 g = gimple_build_cond (LT_EXPR, iter2,
|
|
|
1259 build_int_cst (unsigned_type_node,
|
|
|
1260 node->simdclone->simdlen),
|
|
|
1261 NULL, NULL);
|
|
|
1262 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1263 edge e = split_block (incr_bb, gsi_stmt (gsi));
|
|
|
1264 latch_bb = e->dest;
|
|
|
1265 new_exit_bb = split_block_after_labels (latch_bb)->dest;
|
|
|
1266 loop->latch = latch_bb;
|
|
|
1267
|
|
|
1268 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
|
|
|
1269
|
|
|
1270 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
|
|
|
1271
|
|
|
1272 /* FIXME: Do we need to distribute probabilities for the conditional? */
|
|
|
1273 new_e->probability = profile_probability::guessed_never ();
|
|
|
1274 /* The successor of incr_bb is already pointing to latch_bb; just
|
|
|
1275 change the flags.
|
|
|
1276 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
|
|
|
1277 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
|
|
|
1278 }
|
|
|
1279
|
|
|
1280 gphi *phi = create_phi_node (iter1, body_bb);
|
|
|
1281 edge preheader_edge = find_edge (entry_bb, body_bb);
|
|
|
1282 edge latch_edge = NULL;
|
|
|
1283 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
|
|
|
1284 UNKNOWN_LOCATION);
|
|
|
1285 if (incr_bb)
|
|
|
1286 {
|
|
|
1287 latch_edge = single_succ_edge (latch_bb);
|
|
|
1288 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
|
|
|
1289
|
|
|
1290 /* Generate the new return. */
|
|
|
1291 gsi = gsi_last_bb (new_exit_bb);
|
|
|
1292 if (retval
|
|
|
1293 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
|
|
|
1294 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
|
|
|
1295 retval = TREE_OPERAND (retval, 0);
|
|
|
1296 else if (retval)
|
|
|
1297 {
|
|
|
1298 retval = build1 (VIEW_CONVERT_EXPR,
|
|
|
1299 TREE_TYPE (TREE_TYPE (node->decl)),
|
|
|
1300 retval);
|
|
|
1301 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
|
|
|
1302 false, GSI_CONTINUE_LINKING);
|
|
|
1303 }
|
|
|
1304 g = gimple_build_return (retval);
|
|
|
1305 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
|
|
|
1306 }
|
|
|
1307
|
|
|
1308 /* Handle aligned clauses by replacing default defs of the aligned
|
|
|
1309 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
|
|
|
1310 lhs. Handle linear by adding PHIs. */
|
|
|
1311 for (unsigned i = 0; i < node->simdclone->nargs; i++)
|
|
|
1312 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
|
|
|
1313 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
|
|
|
1314 || !is_gimple_reg_type
|
|
|
1315 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
|
|
|
1316 {
|
|
|
1317 tree orig_arg = node->simdclone->args[i].orig_arg;
|
|
|
1318 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
|
|
|
1319 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
|
|
|
1320 else
|
|
|
1321 {
|
|
|
1322 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
|
|
|
1323 gimple_add_tmp_var (iter1);
|
|
|
1324 }
|
|
|
1325 gsi = gsi_after_labels (entry_bb);
|
|
|
1326 g = gimple_build_assign (iter1, orig_arg);
|
|
|
1327 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1328 gsi = gsi_after_labels (body_bb);
|
|
|
1329 g = gimple_build_assign (orig_arg, iter1);
|
|
|
1330 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1331 }
|
|
|
1332 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
|
|
|
1333 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
|
|
|
1334 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
|
|
|
1335 == REFERENCE_TYPE
|
|
|
1336 && TREE_ADDRESSABLE
|
|
|
1337 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
|
|
|
1338 {
|
|
|
1339 tree orig_arg = node->simdclone->args[i].orig_arg;
|
|
|
1340 tree def = ssa_default_def (cfun, orig_arg);
|
|
|
1341 if (def && !has_zero_uses (def))
|
|
|
1342 {
|
|
|
1343 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
|
|
|
1344 gimple_add_tmp_var (iter1);
|
|
|
1345 gsi = gsi_after_labels (entry_bb);
|
|
|
1346 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
|
|
|
1347 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1348 gsi = gsi_after_labels (body_bb);
|
|
|
1349 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
|
|
|
1350 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1351 }
|
|
|
1352 }
|
|
|
1353 else if (node->simdclone->args[i].alignment
|
|
|
1354 && node->simdclone->args[i].arg_type
|
|
|
1355 == SIMD_CLONE_ARG_TYPE_UNIFORM
|
|
|
1356 && (node->simdclone->args[i].alignment
|
|
|
1357 & (node->simdclone->args[i].alignment - 1)) == 0
|
|
|
1358 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
|
|
|
1359 == POINTER_TYPE)
|
|
|
1360 {
|
|
|
1361 unsigned int alignment = node->simdclone->args[i].alignment;
|
|
|
1362 tree orig_arg = node->simdclone->args[i].orig_arg;
|
|
|
1363 tree def = ssa_default_def (cfun, orig_arg);
|
|
|
1364 if (def && !has_zero_uses (def))
|
|
|
1365 {
|
|
|
1366 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
|
|
|
1367 gimple_seq seq = NULL;
|
|
|
1368 bool need_cvt = false;
|
|
|
1369 gcall *call
|
|
|
1370 = gimple_build_call (fn, 2, def, size_int (alignment));
|
|
|
1371 g = call;
|
|
|
1372 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
|
|
|
1373 ptr_type_node))
|
|
|
1374 need_cvt = true;
|
|
|
1375 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
|
|
|
1376 gimple_call_set_lhs (g, t);
|
|
|
1377 gimple_seq_add_stmt_without_update (&seq, g);
|
|
|
1378 if (need_cvt)
|
|
|
1379 {
|
|
|
1380 t = make_ssa_name (orig_arg);
|
|
|
1381 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
|
|
|
1382 gimple_seq_add_stmt_without_update (&seq, g);
|
|
|
1383 }
|
|
|
1384 gsi_insert_seq_on_edge_immediate
|
|
|
1385 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
|
|
|
1386
|
|
|
1387 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
|
|
|
1388 int freq = compute_call_stmt_bb_frequency (current_function_decl,
|
|
|
1389 entry_bb);
|
|
|
1390 node->create_edge (cgraph_node::get_create (fn),
|
|
|
1391 call, entry_bb->count, freq);
|
|
|
1392
|
|
|
1393 imm_use_iterator iter;
|
|
|
1394 use_operand_p use_p;
|
|
|
1395 gimple *use_stmt;
|
|
|
1396 tree repl = gimple_get_lhs (g);
|
|
|
1397 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
|
|
|
1398 if (is_gimple_debug (use_stmt) || use_stmt == call)
|
|
|
1399 continue;
|
|
|
1400 else
|
|
|
1401 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
|
|
|
1402 SET_USE (use_p, repl);
|
|
|
1403 }
|
|
|
1404 }
|
|
|
1405 else if ((node->simdclone->args[i].arg_type
|
|
|
1406 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
|
|
|
1407 || (node->simdclone->args[i].arg_type
|
|
|
1408 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
|
|
|
1409 || (node->simdclone->args[i].arg_type
|
|
|
1410 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
|
|
|
1411 || (node->simdclone->args[i].arg_type
|
|
|
1412 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
|
|
|
1413 {
|
|
|
1414 tree orig_arg = node->simdclone->args[i].orig_arg;
|
|
|
1415 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
|
|
|
1416 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
|
|
|
1417 tree def = NULL_TREE;
|
|
|
1418 if (TREE_ADDRESSABLE (orig_arg))
|
|
|
1419 {
|
|
|
1420 def = make_ssa_name (TREE_TYPE (orig_arg));
|
|
|
1421 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
|
|
|
1422 if (incr_bb)
|
|
|
1423 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
|
|
|
1424 gsi = gsi_after_labels (entry_bb);
|
|
|
1425 g = gimple_build_assign (def, orig_arg);
|
|
|
1426 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1427 }
|
|
|
1428 else
|
|
|
1429 {
|
|
|
1430 def = ssa_default_def (cfun, orig_arg);
|
|
|
1431 if (!def || has_zero_uses (def))
|
|
|
1432 def = NULL_TREE;
|
|
|
1433 else
|
|
|
1434 {
|
|
|
1435 iter1 = make_ssa_name (orig_arg);
|
|
|
1436 if (incr_bb)
|
|
|
1437 iter2 = make_ssa_name (orig_arg);
|
|
|
1438 }
|
|
|
1439 }
|
|
|
1440 if (def)
|
|
|
1441 {
|
|
|
1442 phi = create_phi_node (iter1, body_bb);
|
|
|
1443 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
|
|
|
1444 if (incr_bb)
|
|
|
1445 {
|
|
|
1446 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
|
|
|
1447 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
|
|
|
1448 ? PLUS_EXPR : POINTER_PLUS_EXPR;
|
|
|
1449 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
|
|
|
1450 ? TREE_TYPE (orig_arg) : sizetype;
|
|
|
1451 tree addcst = simd_clone_linear_addend (node, i, addtype,
|
|
|
1452 entry_bb);
|
|
|
1453 gsi = gsi_last_bb (incr_bb);
|
|
|
1454 g = gimple_build_assign (iter2, code, iter1, addcst);
|
|
|
1455 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1456 }
|
|
|
1457
|
|
|
1458 imm_use_iterator iter;
|
|
|
1459 use_operand_p use_p;
|
|
|
1460 gimple *use_stmt;
|
|
|
1461 if (TREE_ADDRESSABLE (orig_arg))
|
|
|
1462 {
|
|
|
1463 gsi = gsi_after_labels (body_bb);
|
|
|
1464 g = gimple_build_assign (orig_arg, iter1);
|
|
|
1465 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
|
|
|
1466 }
|
|
|
1467 else
|
|
|
1468 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
|
|
|
1469 if (use_stmt == phi)
|
|
|
1470 continue;
|
|
|
1471 else
|
|
|
1472 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
|
|
|
1473 SET_USE (use_p, iter1);
|
|
|
1474 }
|
|
|
1475 }
|
|
|
1476 else if (node->simdclone->args[i].arg_type
|
|
|
1477 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
|
|
|
1478 || (node->simdclone->args[i].arg_type
|
|
|
1479 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
|
|
|
1480 {
|
|
|
1481 tree orig_arg = node->simdclone->args[i].orig_arg;
|
|
|
1482 tree def = ssa_default_def (cfun, orig_arg);
|
|
|
1483 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
|
|
|
1484 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
|
|
|
1485 if (def && !has_zero_uses (def))
|
|
|
1486 {
|
|
|
1487 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
|
|
|
1488 iter1 = make_ssa_name (orig_arg);
|
|
|
1489 if (incr_bb)
|
|
|
1490 iter2 = make_ssa_name (orig_arg);
|
|
|
1491 tree iter3 = make_ssa_name (rtype);
|
|
|
1492 tree iter4 = make_ssa_name (rtype);
|
|
|
1493 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
|
|
|
1494 gsi = gsi_after_labels (entry_bb);
|
|
|
1495 gimple *load
|
|
|
1496 = gimple_build_assign (iter3, build_simple_mem_ref (def));
|
|
|
1497 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
|
|
|
1498
|
|
|
1499 tree array = node->simdclone->args[i].simd_array;
|
|
|
1500 TREE_ADDRESSABLE (array) = 1;
|
|
|
1501 tree ptr = build_fold_addr_expr (array);
|
|
|
1502 phi = create_phi_node (iter1, body_bb);
|
|
|
1503 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
|
|
|
1504 if (incr_bb)
|
|
|
1505 {
|
|
|
1506 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
|
|
|
1507 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
|
|
|
1508 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
|
|
|
1509 gsi = gsi_last_bb (incr_bb);
|
|
|
1510 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1511 }
|
|
|
1512
|
|
|
1513 phi = create_phi_node (iter4, body_bb);
|
|
|
1514 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
|
|
|
1515 if (incr_bb)
|
|
|
1516 {
|
|
|
1517 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
|
|
|
1518 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
|
|
|
1519 ? PLUS_EXPR : POINTER_PLUS_EXPR;
|
|
|
1520 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
|
|
|
1521 ? TREE_TYPE (iter3) : sizetype;
|
|
|
1522 tree addcst = simd_clone_linear_addend (node, i, addtype,
|
|
|
1523 entry_bb);
|
|
|
1524 g = gimple_build_assign (iter5, code, iter4, addcst);
|
|
|
1525 gsi = gsi_last_bb (incr_bb);
|
|
|
1526 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1527 }
|
|
|
1528
|
|
|
1529 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
|
|
|
1530 gsi = gsi_after_labels (body_bb);
|
|
|
1531 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1532
|
|
|
1533 imm_use_iterator iter;
|
|
|
1534 use_operand_p use_p;
|
|
|
1535 gimple *use_stmt;
|
|
|
1536 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
|
|
|
1537 if (use_stmt == load)
|
|
|
1538 continue;
|
|
|
1539 else
|
|
|
1540 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
|
|
|
1541 SET_USE (use_p, iter1);
|
|
|
1542
|
|
|
1543 if (!TYPE_READONLY (rtype) && incr_bb)
|
|
|
1544 {
|
|
|
1545 tree v = make_ssa_name (rtype);
|
|
|
1546 tree aref = build4 (ARRAY_REF, rtype, array,
|
|
|
1547 size_zero_node, NULL_TREE,
|
|
|
1548 NULL_TREE);
|
|
|
1549 gsi = gsi_after_labels (new_exit_bb);
|
|
|
1550 g = gimple_build_assign (v, aref);
|
|
|
1551 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1552 g = gimple_build_assign (build_simple_mem_ref (def), v);
|
|
|
1553 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
|
|
|
1554 }
|
|
|
1555 }
|
|
|
1556 }
|
|
|
1557
|
|
|
1558 calculate_dominance_info (CDI_DOMINATORS);
|
|
|
1559 if (loop)
|
|
|
1560 add_loop (loop, loop->header->loop_father);
|
|
|
1561 update_ssa (TODO_update_ssa);
|
|
|
1562
|
|
|
1563 pop_cfun ();
|
|
|
1564 }
|
|
|
1565
|
|
|
1566 /* If the function in NODE is tagged as an elemental SIMD function,
|
|
|
1567 create the appropriate SIMD clones. */
|
|
|
1568
|
|
|
1569 static void
|
|
|
1570 expand_simd_clones (struct cgraph_node *node)
|
|
|
1571 {
|
|
|
1572 tree attr = lookup_attribute ("omp declare simd",
|
|
|
1573 DECL_ATTRIBUTES (node->decl));
|
|
|
1574 if (attr == NULL_TREE
|
|
|
1575 || node->global.inlined_to
|
|
|
1576 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
|
|
|
1577 return;
|
|
|
1578
|
|
|
1579 /* Ignore
|
|
|
1580 #pragma omp declare simd
|
|
|
1581 extern int foo ();
|
|
|
1582 in C, there we don't know the argument types at all. */
|
|
|
1583 if (!node->definition
|
|
|
1584 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
|
|
|
1585 return;
|
|
|
1586
|
|
|
1587 /* Call this before creating clone_info, as it might ggc_collect. */
|
|
|
1588 if (node->definition && node->has_gimple_body_p ())
|
|
|
1589 node->get_body ();
|
|
|
1590
|
|
|
1591 do
|
|
|
1592 {
|
|
|
1593 /* Start with parsing the "omp declare simd" attribute(s). */
|
|
|
1594 bool inbranch_clause_specified;
|
|
|
1595 struct cgraph_simd_clone *clone_info
|
|
|
1596 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
|
|
|
1597 &inbranch_clause_specified);
|
|
|
1598 if (clone_info == NULL)
|
|
|
1599 continue;
|
|
|
1600
|
|
|
1601 int orig_simdlen = clone_info->simdlen;
|
|
|
1602 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
|
|
|
1603 /* The target can return 0 (no simd clones should be created),
|
|
|
1604 1 (just one ISA of simd clones should be created) or higher
|
|
|
1605 count of ISA variants. In that case, clone_info is initialized
|
|
|
1606 for the first ISA variant. */
|
|
|
1607 int count
|
|
|
1608 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
|
|
|
1609 base_type, 0);
|
|
|
1610 if (count == 0)
|
|
|
1611 continue;
|
|
|
1612
|
|
|
1613 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
|
|
|
1614 also create one inbranch and one !inbranch clone of it. */
|
|
|
1615 for (int i = 0; i < count * 2; i++)
|
|
|
1616 {
|
|
|
1617 struct cgraph_simd_clone *clone = clone_info;
|
|
|
1618 if (inbranch_clause_specified && (i & 1) != 0)
|
|
|
1619 continue;
|
|
|
1620
|
|
|
1621 if (i != 0)
|
|
|
1622 {
|
|
|
1623 clone = simd_clone_struct_alloc (clone_info->nargs
|
|
|
1624 + ((i & 1) != 0));
|
|
|
1625 simd_clone_struct_copy (clone, clone_info);
|
|
|
1626 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
|
|
|
1627 and simd_clone_adjust_argument_types did to the first
|
|
|
1628 clone's info. */
|
|
|
1629 clone->nargs -= clone_info->inbranch;
|
|
|
1630 clone->simdlen = orig_simdlen;
|
|
|
1631 /* And call the target hook again to get the right ISA. */
|
|
|
1632 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
|
|
|
1633 base_type,
|
|
|
1634 i / 2);
|
|
|
1635 if ((i & 1) != 0)
|
|
|
1636 clone->inbranch = 1;
|
|
|
1637 }
|
|
|
1638
|
|
|
1639 /* simd_clone_mangle might fail if such a clone has been created
|
|
|
1640 already. */
|
|
|
1641 tree id = simd_clone_mangle (node, clone);
|
|
|
1642 if (id == NULL_TREE)
|
|
|
1643 continue;
|
|
|
1644
|
|
|
1645 /* Only when we are sure we want to create the clone actually
|
|
|
1646 clone the function (or definitions) or create another
|
|
|
1647 extern FUNCTION_DECL (for prototypes without definitions). */
|
|
|
1648 struct cgraph_node *n = simd_clone_create (node);
|
|
|
1649 if (n == NULL)
|
|
|
1650 continue;
|
|
|
1651
|
|
|
1652 n->simdclone = clone;
|
|
|
1653 clone->origin = node;
|
|
|
1654 clone->next_clone = NULL;
|
|
|
1655 if (node->simd_clones == NULL)
|
|
|
1656 {
|
|
|
1657 clone->prev_clone = n;
|
|
|
1658 node->simd_clones = n;
|
|
|
1659 }
|
|
|
1660 else
|
|
|
1661 {
|
|
|
1662 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
|
|
|
1663 clone->prev_clone->simdclone->next_clone = n;
|
|
|
1664 node->simd_clones->simdclone->prev_clone = n;
|
|
|
1665 }
|
|
|
1666 symtab->change_decl_assembler_name (n->decl, id);
|
|
|
1667 /* And finally adjust the return type, parameters and for
|
|
|
1668 definitions also function body. */
|
|
|
1669 if (node->definition)
|
|
|
1670 simd_clone_adjust (n);
|
|
|
1671 else
|
|
|
1672 {
|
|
|
1673 simd_clone_adjust_return_type (n);
|
|
|
1674 simd_clone_adjust_argument_types (n);
|
|
|
1675 }
|
|
|
1676 }
|
|
|
1677 }
|
|
|
1678 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
|
|
|
1679 }
|
|
|
1680
|
|
|
1681 /* Entry point for IPA simd clone creation pass. */
|
|
|
1682
|
|
|
1683 static unsigned int
|
|
|
1684 ipa_omp_simd_clone (void)
|
|
|
1685 {
|
|
|
1686 struct cgraph_node *node;
|
|
|
1687 FOR_EACH_FUNCTION (node)
|
|
|
1688 expand_simd_clones (node);
|
|
|
1689 return 0;
|
|
|
1690 }
|
|
|
1691
|
|
|
1692 namespace {
|
|
|
1693
|
|
|
1694 const pass_data pass_data_omp_simd_clone =
|
|
|
1695 {
|
|
|
1696 SIMPLE_IPA_PASS, /* type */
|
|
|
1697 "simdclone", /* name */
|
|
|
1698 OPTGROUP_OMP, /* optinfo_flags */
|
|
|
1699 TV_NONE, /* tv_id */
|
|
|
1700 ( PROP_ssa | PROP_cfg ), /* properties_required */
|
|
|
1701 0, /* properties_provided */
|
|
|
1702 0, /* properties_destroyed */
|
|
|
1703 0, /* todo_flags_start */
|
|
|
1704 0, /* todo_flags_finish */
|
|
|
1705 };
|
|
|
1706
|
|
|
1707 class pass_omp_simd_clone : public simple_ipa_opt_pass
|
|
|
1708 {
|
|
|
1709 public:
|
|
|
1710 pass_omp_simd_clone(gcc::context *ctxt)
|
|
|
1711 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
|
|
|
1712 {}
|
|
|
1713
|
|
|
1714 /* opt_pass methods: */
|
|
|
1715 virtual bool gate (function *);
|
|
|
1716 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
|
|
|
1717 };
|
|
|
1718
|
|
|
1719 bool
|
|
|
1720 pass_omp_simd_clone::gate (function *)
|
|
|
1721 {
|
|
|
1722 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
|
|
|
1723 }
|
|
|
1724
|
|
|
1725 } // anon namespace
|
|
|
1726
|
|
|
1727 simple_ipa_opt_pass *
|
|
|
1728 make_pass_omp_simd_clone (gcc::context *ctxt)
|
|
|
1729 {
|
|
|
1730 return new pass_omp_simd_clone (ctxt);
|
|
|
1731 }
|
