Mercurial > hg > CbC > CbC_gcc

comparison gcc/tree-vectorizer.h @ 55:77e2b8dfacca gcc-4.4.5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
update it from 4.4.3 to 4.5.0
author ryoma <e075725@ie.u-ryukyu.ac.jp>
date Fri, 12 Feb 2010 23:39:51 +0900
parents a06113de4d67
children b7f97abdc517
comparison
equal deleted inserted replaced
52:c156f1bd5cd9 55:77e2b8dfacca
1 /* Loop Vectorization 1 /* Vectorizer
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. 2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free
3 Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com> 4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 5
5 This file is part of GCC. 6 This file is part of GCC.
6 7
7 GCC is free software; you can redistribute it and/or modify it under 8 GCC is free software; you can redistribute it and/or modify it under
18 along with GCC; see the file COPYING3. If not see 19 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */ 20 <http://www.gnu.org/licenses/>. */
20 21
21 #ifndef GCC_TREE_VECTORIZER_H 22 #ifndef GCC_TREE_VECTORIZER_H
22 #define GCC_TREE_VECTORIZER_H 23 #define GCC_TREE_VECTORIZER_H
24
25 #include "tree-data-ref.h"
23 26
24 typedef source_location LOC; 27 typedef source_location LOC;
25 #define UNKNOWN_LOC UNKNOWN_LOCATION 28 #define UNKNOWN_LOC UNKNOWN_LOCATION
26 #define EXPR_LOC(e) EXPR_LOCATION(e) 29 #define EXPR_LOC(e) EXPR_LOCATION(e)
27 #define LOC_FILE(l) LOCATION_FILE (l) 30 #define LOC_FILE(l) LOCATION_FILE (l)
50 dr_aligned 53 dr_aligned
51 }; 54 };
52 55
53 /* Define type of def-use cross-iteration cycle. */ 56 /* Define type of def-use cross-iteration cycle. */
54 enum vect_def_type { 57 enum vect_def_type {
58 vect_uninitialized_def = 0,
55 vect_constant_def = 1, 59 vect_constant_def = 1,
56 vect_invariant_def, 60 vect_external_def,
57 vect_loop_def, 61 vect_internal_def,
58 vect_induction_def, 62 vect_induction_def,
59 vect_reduction_def, 63 vect_reduction_def,
64 vect_double_reduction_def,
65 vect_nested_cycle,
60 vect_unknown_def_type 66 vect_unknown_def_type
61 }; 67 };
62 68
63 /* Define verbosity levels. */ 69 /* Define verbosity levels. */
64 enum verbosity_levels { 70 enum verbosity_levels {
65 REPORT_NONE, 71 REPORT_NONE,
66 REPORT_VECTORIZED_LOOPS, 72 REPORT_VECTORIZED_LOCATIONS,
67 REPORT_UNVECTORIZED_LOOPS, 73 REPORT_UNVECTORIZED_LOCATIONS,
68 REPORT_COST, 74 REPORT_COST,
69 REPORT_ALIGNMENT, 75 REPORT_ALIGNMENT,
70 REPORT_DR_DETAILS, 76 REPORT_DR_DETAILS,
71 REPORT_BAD_FORM_LOOPS, 77 REPORT_BAD_FORM_LOOPS,
72 REPORT_OUTER_LOOPS, 78 REPORT_OUTER_LOOPS,
90 struct _slp_tree *right; 96 struct _slp_tree *right;
91 /* A group of scalar stmts to be vectorized together. */ 97 /* A group of scalar stmts to be vectorized together. */
92 VEC (gimple, heap) *stmts; 98 VEC (gimple, heap) *stmts;
93 /* Vectorized stmt/s. */ 99 /* Vectorized stmt/s. */
94 VEC (gimple, heap) *vec_stmts; 100 VEC (gimple, heap) *vec_stmts;
95 /* Number of vector stmts that are created to replace the group of scalar 101 /* Number of vector stmts that are created to replace the group of scalar
96 stmts. It is calculated during the transformation phase as the number of 102 stmts. It is calculated during the transformation phase as the number of
97 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF 103 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
98 divided by vector size. */ 104 divided by vector size. */
99 unsigned int vec_stmts_size; 105 unsigned int vec_stmts_size;
100 /* Vectorization costs associated with SLP node. */ 106 /* Vectorization costs associated with SLP node. */
101 struct 107 struct
102 { 108 {
119 125
120 /* The unrolling factor required to vectorized this SLP instance. */ 126 /* The unrolling factor required to vectorized this SLP instance. */
121 unsigned int unrolling_factor; 127 unsigned int unrolling_factor;
122 128
123 /* Vectorization costs associated with SLP instance. */ 129 /* Vectorization costs associated with SLP instance. */
124 struct 130 struct
125 { 131 {
126 int outside_of_loop; /* Statements generated outside loop. */ 132 int outside_of_loop; /* Statements generated outside loop. */
127 int inside_of_loop; /* Statements generated inside loop. */ 133 int inside_of_loop; /* Statements generated inside loop. */
128 } cost; 134 } cost;
129 135
130 /* Loads permutation relatively to the stores, NULL if there is no 136 /* Loads permutation relatively to the stores, NULL if there is no
131 permutation. */ 137 permutation. */
132 VEC (int, heap) *load_permutation; 138 VEC (int, heap) *load_permutation;
133 139
134 /* The group of nodes that contain loads of this SLP instance. */ 140 /* The group of nodes that contain loads of this SLP instance. */
135 VEC (slp_tree, heap) *loads; 141 VEC (slp_tree, heap) *loads;
174 /* Number of iterations. */ 180 /* Number of iterations. */
175 tree num_iters; 181 tree num_iters;
176 tree num_iters_unchanged; 182 tree num_iters_unchanged;
177 183
178 /* Minimum number of iterations below which vectorization is expected to 184 /* Minimum number of iterations below which vectorization is expected to
179 not be profitable (as estimated by the cost model). 185 not be profitable (as estimated by the cost model).
180 -1 indicates that vectorization will not be profitable. 186 -1 indicates that vectorization will not be profitable.
181 FORNOW: This field is an int. Will be a tree in the future, to represent 187 FORNOW: This field is an int. Will be a tree in the future, to represent
182 values unknown at compile time. */ 188 values unknown at compile time. */
183 int min_profitable_iters; 189 int min_profitable_iters;
184 190
185 /* Is the loop vectorizable? */ 191 /* Is the loop vectorizable? */
186 bool vectorizable; 192 bool vectorizable;
187 193
188 /* Unrolling factor */ 194 /* Unrolling factor */
189 int vectorization_factor; 195 int vectorization_factor;
227 233
228 /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES 234 /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES
229 of the loop. */ 235 of the loop. */
230 VEC(slp_instance, heap) *slp_instances; 236 VEC(slp_instance, heap) *slp_instances;
231 237
232 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is 238 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
233 applied to the loop, i.e., no unrolling is needed, this is 1. */ 239 applied to the loop, i.e., no unrolling is needed, this is 1. */
234 unsigned slp_unrolling_factor; 240 unsigned slp_unrolling_factor;
235 } *loop_vec_info; 241 } *loop_vec_info;
236 242
237 /* Access Functions. */ 243 /* Access Functions. */
238 #define LOOP_VINFO_LOOP(L) (L)->loop 244 #define LOOP_VINFO_LOOP(L) (L)->loop
239 #define LOOP_VINFO_BBS(L) (L)->bbs 245 #define LOOP_VINFO_BBS(L) (L)->bbs
240 #define LOOP_VINFO_NITERS(L) (L)->num_iters 246 #define LOOP_VINFO_NITERS(L) (L)->num_iters
241 /* Since LOOP_VINFO_NITERS can change after prologue peeling 247 /* Since LOOP_VINFO_NITERS can change after prologue peeling
242 retain total unchanged scalar loop iterations for cost model. */ 248 retain total unchanged scalar loop iterations for cost model. */
243 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged 249 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
244 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters 250 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
245 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable 251 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
246 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor 252 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
247 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask 253 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
248 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs 254 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs
249 #define LOOP_VINFO_DDRS(L) (L)->ddrs 255 #define LOOP_VINFO_DDRS(L) (L)->ddrs
250 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters)) 256 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
251 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment 257 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
252 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr 258 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
253 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts 259 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
254 #define LOOP_VINFO_LOC(L) (L)->loop_line_number 260 #define LOOP_VINFO_LOC(L) (L)->loop_line_number
255 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs 261 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
256 #define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores 262 #define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
257 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances 263 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
258 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor 264 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
265
266 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
267 VEC_length (gimple, (L)->may_misalign_stmts) > 0
268 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
269 VEC_length (ddr_p, (L)->may_alias_ddrs) > 0
259 270
260 #define NITERS_KNOWN_P(n) \ 271 #define NITERS_KNOWN_P(n) \
261 (host_integerp ((n),0) \ 272 (host_integerp ((n),0) \
262 && TREE_INT_CST_LOW ((n)) > 0) 273 && TREE_INT_CST_LOW ((n)) > 0)
263 274
264 #define LOOP_VINFO_NITERS_KNOWN_P(L) \ 275 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
265 NITERS_KNOWN_P((L)->num_iters) 276 NITERS_KNOWN_P((L)->num_iters)
266 277
267 static inline loop_vec_info 278 static inline loop_vec_info
268 loop_vec_info_for_loop (struct loop *loop) 279 loop_vec_info_for_loop (struct loop *loop)
269 { 280 {
271 } 282 }
272 283
273 static inline bool 284 static inline bool
274 nested_in_vect_loop_p (struct loop *loop, gimple stmt) 285 nested_in_vect_loop_p (struct loop *loop, gimple stmt)
275 { 286 {
276 return (loop->inner 287 return (loop->inner
277 && (loop->inner == (gimple_bb (stmt))->loop_father)); 288 && (loop->inner == (gimple_bb (stmt))->loop_father));
289 }
290
291 typedef struct _bb_vec_info {
292
293 basic_block bb;
294 /* All interleaving chains of stores in the basic block, represented by the
295 first stmt in the chain. */
296 VEC(gimple, heap) *strided_stores;
297
298 /* All SLP instances in the basic block. This is a subset of the set of
299 STRIDED_STORES of the basic block. */
300 VEC(slp_instance, heap) *slp_instances;
301
302 /* All data references in the basic block. */
303 VEC (data_reference_p, heap) *datarefs;
304
305 /* All data dependences in the basic block. */
306 VEC (ddr_p, heap) *ddrs;
307 } *bb_vec_info;
308
309 #define BB_VINFO_BB(B) (B)->bb
310 #define BB_VINFO_STRIDED_STORES(B) (B)->strided_stores
311 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
312 #define BB_VINFO_DATAREFS(B) (B)->datarefs
313 #define BB_VINFO_DDRS(B) (B)->ddrs
314
315 static inline bb_vec_info
316 vec_info_for_bb (basic_block bb)
317 {
318 return (bb_vec_info) bb->aux;
278 } 319 }
279 320
280 /*-----------------------------------------------------------------*/ 321 /*-----------------------------------------------------------------*/
281 /* Info on vectorized defs. */ 322 /* Info on vectorized defs. */
282 /*-----------------------------------------------------------------*/ 323 /*-----------------------------------------------------------------*/
294 type_demotion_vec_info_type, 335 type_demotion_vec_info_type,
295 type_conversion_vec_info_type, 336 type_conversion_vec_info_type,
296 loop_exit_ctrl_vec_info_type 337 loop_exit_ctrl_vec_info_type
297 }; 338 };
298 339
299 /* Indicates whether/how a variable is used in the loop. */ 340 /* Indicates whether/how a variable is used in the scope of loop/basic
341 block. */
300 enum vect_relevant { 342 enum vect_relevant {
301 vect_unused_in_loop = 0, 343 vect_unused_in_scope = 0,
344 /* The def is in the inner loop, and the use is in the outer loop, and the
345 use is a reduction stmt. */
302 vect_used_in_outer_by_reduction, 346 vect_used_in_outer_by_reduction,
347 /* The def is in the inner loop, and the use is in the outer loop (and is
348 not part of reduction). */
303 vect_used_in_outer, 349 vect_used_in_outer,
304 350
305 /* defs that feed computations that end up (only) in a reduction. These 351 /* defs that feed computations that end up (only) in a reduction. These
306 defs may be used by non-reduction stmts, but eventually, any 352 defs may be used by non-reduction stmts, but eventually, any
307 computations/values that are affected by these defs are used to compute 353 computations/values that are affected by these defs are used to compute
308 a reduction (i.e. don't get stored to memory, for example). We use this 354 a reduction (i.e. don't get stored to memory, for example). We use this
309 to identify computations that we can change the order in which they are 355 to identify computations that we can change the order in which they are
310 computed. */ 356 computed. */
311 vect_used_by_reduction, 357 vect_used_by_reduction,
312 358
313 vect_used_in_loop 359 vect_used_in_scope
314 }; 360 };
315 361
316 /* The type of vectorization that can be applied to the stmt: regular loop-based 362 /* The type of vectorization that can be applied to the stmt: regular loop-based
317 vectorization; pure SLP - the stmt is a part of SLP instances and does not 363 vectorization; pure SLP - the stmt is a part of SLP instances and does not
318 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is 364 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
319 a part of SLP instance and also must be loop-based vectorized, since it has 365 a part of SLP instance and also must be loop-based vectorized, since it has
320 uses outside SLP sequences. 366 uses outside SLP sequences.
321 367
322 In the loop context the meanings of pure and hybrid SLP are slightly 368 In the loop context the meanings of pure and hybrid SLP are slightly
323 different. By saying that pure SLP is applied to the loop, we mean that we 369 different. By saying that pure SLP is applied to the loop, we mean that we
324 exploit only intra-iteration parallelism in the loop; i.e., the loop can be 370 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
325 vectorized without doing any conceptual unrolling, cause we don't pack 371 vectorized without doing any conceptual unrolling, cause we don't pack
326 together stmts from different iterations, only within a single iteration. 372 together stmts from different iterations, only within a single iteration.
327 Loop hybrid SLP means that we exploit both intra-iteration and 373 Loop hybrid SLP means that we exploit both intra-iteration and
328 inter-iteration parallelism (e.g., number of elements in the vector is 4 374 inter-iteration parallelism (e.g., number of elements in the vector is 4
329 and the slp-group-size is 2, in which case we don't have enough parallelism 375 and the slp-group-size is 2, in which case we don't have enough parallelism
330 within an iteration, so we obtain the rest of the parallelism from subsequent 376 within an iteration, so we obtain the rest of the parallelism from subsequent
331 iterations by unrolling the loop by 2). */ 377 iterations by unrolling the loop by 2). */
332 enum slp_vect_type { 378 enum slp_vect_type {
333 loop_vect = 0, 379 loop_vect = 0,
334 pure_slp, 380 pure_slp,
335 hybrid 381 hybrid
336 }; 382 };
337 383
365 /* The vectorized version of the stmt. */ 411 /* The vectorized version of the stmt. */
366 gimple vectorized_stmt; 412 gimple vectorized_stmt;
367 413
368 414
369 /** The following is relevant only for stmts that contain a non-scalar 415 /** The following is relevant only for stmts that contain a non-scalar
370 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have 416 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
371 at most one such data-ref. **/ 417 at most one such data-ref. **/
372 418
373 /* Information about the data-ref (access function, etc), 419 /* Information about the data-ref (access function, etc),
374 relative to the inner-most containing loop. */ 420 relative to the inner-most containing loop. */
375 struct data_reference *data_ref_info; 421 struct data_reference *data_ref_info;
383 tree dr_aligned_to; 429 tree dr_aligned_to;
384 430
385 /* Stmt is part of some pattern (computation idiom) */ 431 /* Stmt is part of some pattern (computation idiom) */
386 bool in_pattern_p; 432 bool in_pattern_p;
387 433
388 /* Used for various bookkeeping purposes, generally holding a pointer to 434 /* Used for various bookkeeping purposes, generally holding a pointer to
389 some other stmt S that is in some way "related" to this stmt. 435 some other stmt S that is in some way "related" to this stmt.
390 Current use of this field is: 436 Current use of this field is:
391 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is 437 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
392 true): S is the "pattern stmt" that represents (and replaces) the 438 true): S is the "pattern stmt" that represents (and replaces) the
393 sequence of stmts that constitutes the pattern. Similarly, the 439 sequence of stmts that constitutes the pattern. Similarly, the
394 related_stmt of the "pattern stmt" points back to this stmt (which is 440 related_stmt of the "pattern stmt" points back to this stmt (which is
395 the last stmt in the original sequence of stmts that constitutes the 441 the last stmt in the original sequence of stmts that constitutes the
396 pattern). */ 442 pattern). */
397 gimple related_stmt; 443 gimple related_stmt;
398 444
399 /* List of datarefs that are known to have the same alignment as the dataref 445 /* List of datarefs that are known to have the same alignment as the dataref
400 of this stmt. */ 446 of this stmt. */
422 /* For loads only, if there is a store with the same location, this field is 468 /* For loads only, if there is a store with the same location, this field is
423 TRUE. */ 469 TRUE. */
424 bool read_write_dep; 470 bool read_write_dep;
425 471
426 /* Vectorization costs associated with statement. */ 472 /* Vectorization costs associated with statement. */
427 struct 473 struct
428 { 474 {
429 int outside_of_loop; /* Statements generated outside loop. */ 475 int outside_of_loop; /* Statements generated outside loop. */
430 int inside_of_loop; /* Statements generated inside loop. */ 476 int inside_of_loop; /* Statements generated inside loop. */
431 } cost; 477 } cost;
432 478
433 /* Whether the stmt is SLPed, loop-based vectorized, or both. */ 479 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
434 enum slp_vect_type slp_type; 480 enum slp_vect_type slp_type;
481
482 /* The bb_vec_info with respect to which STMT is vectorized. */
483 bb_vec_info bb_vinfo;
435 } *stmt_vec_info; 484 } *stmt_vec_info;
436 485
437 /* Access Functions. */ 486 /* Access Functions. */
438 #define STMT_VINFO_TYPE(S) (S)->type 487 #define STMT_VINFO_TYPE(S) (S)->type
439 #define STMT_VINFO_STMT(S) (S)->stmt 488 #define STMT_VINFO_STMT(S) (S)->stmt
440 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo 489 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo
490 #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo
441 #define STMT_VINFO_RELEVANT(S) (S)->relevant 491 #define STMT_VINFO_RELEVANT(S) (S)->relevant
442 #define STMT_VINFO_LIVE_P(S) (S)->live 492 #define STMT_VINFO_LIVE_P(S) (S)->live
443 #define STMT_VINFO_VECTYPE(S) (S)->vectype 493 #define STMT_VINFO_VECTYPE(S) (S)->vectype
444 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt 494 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
445 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info 495 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info
469 #define DR_GROUP_STORE_COUNT(S) (S)->store_count 519 #define DR_GROUP_STORE_COUNT(S) (S)->store_count
470 #define DR_GROUP_GAP(S) (S)->gap 520 #define DR_GROUP_GAP(S) (S)->gap
471 #define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt 521 #define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
472 #define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep 522 #define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
473 523
474 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_loop) 524 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
475 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop 525 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
476 #define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop 526 #define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
477 527
478 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid) 528 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
479 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp) 529 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
506 #ifndef TARG_SCALAR_STORE_COST 556 #ifndef TARG_SCALAR_STORE_COST
507 #define TARG_SCALAR_STORE_COST 1 557 #define TARG_SCALAR_STORE_COST 1
508 #endif 558 #endif
509 559
510 /* Cost of any vector operation, excluding load, store or vector to scalar 560 /* Cost of any vector operation, excluding load, store or vector to scalar
511 operation. */ 561 operation. */
512 #ifndef TARG_VEC_STMT_COST 562 #ifndef TARG_VEC_STMT_COST
513 #define TARG_VEC_STMT_COST 1 563 #define TARG_VEC_STMT_COST 1
514 #endif 564 #endif
515 565
516 /* Cost of vector to scalar operation. */ 566 /* Cost of vector to scalar operation. */
631 return true; 681 return true;
632 gcc_assert (EDGE_COUNT (bb->preds) == 1); 682 gcc_assert (EDGE_COUNT (bb->preds) == 1);
633 return false; 683 return false;
634 } 684 }
635 685
636 static inline void 686 static inline void
637 stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node, 687 stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
638 int cost) 688 int cost)
639 { 689 {
640 if (slp_node) 690 if (slp_node)
641 SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost; 691 SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost;
642 else 692 else
643 STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost; 693 STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost;
644 } 694 }
645 695
646 static inline void 696 static inline void
647 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node, 697 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
648 int cost) 698 int cost)
649 { 699 {
650 if (slp_node) 700 if (slp_node)
651 SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost; 701 SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost;
652 else 702 else
653 STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost; 703 STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost;
654 } 704 }
655 705
656 static inline int 706 static inline int
657 vect_pow2 (int x) 707 vect_pow2 (int x)
658 { 708 {
659 int i, res = 1; 709 int i, res = 1;
685 return (DR_MISALIGNMENT (data_ref_info) != -1); 735 return (DR_MISALIGNMENT (data_ref_info) != -1);
686 } 736 }
687 737
688 /* vect_dump will be set to stderr or dump_file if exist. */ 738 /* vect_dump will be set to stderr or dump_file if exist. */
689 extern FILE *vect_dump; 739 extern FILE *vect_dump;
690 extern enum verbosity_levels vect_verbosity_level; 740 extern LOC vect_loop_location;
691
692 /* Bitmap of virtual variables to be renamed. */
693 extern bitmap vect_memsyms_to_rename;
694 741
695 /*-----------------------------------------------------------------*/ 742 /*-----------------------------------------------------------------*/
696 /* Function prototypes. */ 743 /* Function prototypes. */
697 /*-----------------------------------------------------------------*/ 744 /*-----------------------------------------------------------------*/
698 745
699 /************************************************************************* 746 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
700 Simple Loop Peeling Utilities - in tree-vectorizer.c 747 in tree-vect-loop-manip.c. */
701 *************************************************************************/
702 /* Entry point for peeling of simple loops.
703 Peel the first/last iterations of a loop.
704 It can be used outside of the vectorizer for loops that are simple enough
705 (see function documentation). In the vectorizer it is used to peel the
706 last few iterations when the loop bound is unknown or does not evenly
707 divide by the vectorization factor, and to peel the first few iterations
708 to force the alignment of data references in the loop. */
709 extern struct loop *slpeel_tree_peel_loop_to_edge
710 (struct loop *, edge, tree, tree, bool, unsigned int, bool);
711 extern void set_prologue_iterations (basic_block, tree,
712 struct loop *, unsigned int);
713 struct loop *tree_duplicate_loop_on_edge (struct loop *, edge);
714 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree); 748 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
715 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge); 749 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
716 #ifdef ENABLE_CHECKING 750 extern void vect_loop_versioning (loop_vec_info, bool, tree *, gimple_seq *);
717 extern void slpeel_verify_cfg_after_peeling (struct loop *, struct loop *); 751 extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
718 #endif 752 tree, gimple_seq);
719 753 extern void vect_do_peeling_for_alignment (loop_vec_info);
720 754 extern LOC find_loop_location (struct loop *);
721 /************************************************************************* 755 extern bool vect_can_advance_ivs_p (loop_vec_info);
722 General Vectorization Utilities 756
723 *************************************************************************/ 757 /* In tree-vect-stmts.c. */
724 /** In tree-vectorizer.c **/
725 extern tree get_vectype_for_scalar_type (tree); 758 extern tree get_vectype_for_scalar_type (tree);
726 extern bool vect_is_simple_use (tree, loop_vec_info, gimple *, tree *, 759 extern bool vect_is_simple_use (tree, loop_vec_info, bb_vec_info, gimple *,
727 enum vect_def_type *); 760 tree *, enum vect_def_type *);
728 extern bool vect_is_simple_iv_evolution (unsigned, tree, tree *, tree *); 761 extern bool supportable_widening_operation (enum tree_code, gimple, tree,
729 extern gimple vect_is_simple_reduction (loop_vec_info, gimple); 762 tree *, tree *, enum tree_code *,
763 enum tree_code *, int *,
764 VEC (tree, heap) **);
765 extern bool supportable_narrowing_operation (enum tree_code, const_gimple,
766 tree, enum tree_code *, int *,
767 VEC (tree, heap) **);
768 extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info,
769 bb_vec_info);
770 extern void free_stmt_vec_info (gimple stmt);
771 extern tree vectorizable_function (gimple, tree, tree);
772 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
773 slp_tree);
774 extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type,
775 slp_tree);
776 extern void vect_model_load_cost (stmt_vec_info, int, slp_tree);
777 extern void vect_finish_stmt_generation (gimple, gimple,
778 gimple_stmt_iterator *);
779 extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
780 extern int cost_for_stmt (gimple);
781 extern tree vect_get_vec_def_for_operand (tree, gimple, tree *);
782 extern tree vect_init_vector (gimple, tree, tree,
783 gimple_stmt_iterator *);
784 extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree);
785 extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *,
786 bool *, slp_tree, slp_instance);
787 extern void vect_remove_stores (gimple);
788 extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
789 extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
790 tree, int);
791
792 /* In tree-vect-data-refs.c. */
730 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int); 793 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
731 extern enum dr_alignment_support vect_supportable_dr_alignment 794 extern enum dr_alignment_support vect_supportable_dr_alignment
732 (struct data_reference *); 795 (struct data_reference *);
733 extern bool reduction_code_for_scalar_code (enum tree_code, enum tree_code *); 796 extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *,
734 extern bool supportable_widening_operation (enum tree_code, gimple, tree, 797 HOST_WIDE_INT *);
735 tree *, tree *, enum tree_code *, enum tree_code *, 798 extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info);
736 int *, VEC (tree, heap) **); 799 extern bool vect_enhance_data_refs_alignment (loop_vec_info);
737 extern bool supportable_narrowing_operation (enum tree_code, const_gimple, 800 extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info);
738 tree, enum tree_code *, int *, VEC (tree, heap) **); 801 extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info);
739 802 extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info);
740 /* Creation and deletion of loop and stmt info structs. */ 803 extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
741 extern loop_vec_info new_loop_vec_info (struct loop *loop); 804 extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info);
805 extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *,
806 gimple *, bool, bool *);
807 extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);
808 extern tree vect_create_destination_var (tree, tree);
809 extern bool vect_strided_store_supported (tree);
810 extern bool vect_strided_load_supported (tree);
811 extern bool vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
812 gimple_stmt_iterator *, VEC(tree,heap) **);
813 extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *,
814 enum dr_alignment_support, tree,
815 struct loop **);
816 extern bool vect_permute_load_chain (VEC(tree,heap) *,unsigned int, gimple,
817 gimple_stmt_iterator *, VEC(tree,heap) **);
818 extern bool vect_transform_strided_load (gimple, VEC(tree,heap) *, int,
819 gimple_stmt_iterator *);
820 extern int vect_get_place_in_interleaving_chain (gimple, gimple);
821 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
822 extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *,
823 tree, struct loop *);
824
825 /* In tree-vect-loop.c. */
826 /* FORNOW: Used in tree-parloops.c. */
742 extern void destroy_loop_vec_info (loop_vec_info, bool); 827 extern void destroy_loop_vec_info (loop_vec_info, bool);
743 extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info); 828 extern gimple vect_is_simple_reduction (loop_vec_info, gimple, bool, bool *);
744 extern void free_stmt_vec_info (gimple stmt); 829 /* Drive for loop analysis stage. */
745
746
747 /** In tree-vect-analyze.c **/
748 /* Driver for analysis stage. */
749 extern loop_vec_info vect_analyze_loop (struct loop *); 830 extern loop_vec_info vect_analyze_loop (struct loop *);
831 /* Drive for loop transformation stage. */
832 extern void vect_transform_loop (loop_vec_info);
833 extern loop_vec_info vect_analyze_loop_form (struct loop *);
834 extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *,
835 gimple *);
836 extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *);
837 extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *);
838 extern int vect_estimate_min_profitable_iters (loop_vec_info);
839 extern tree get_initial_def_for_reduction (gimple, tree, tree *);
840 extern int vect_min_worthwhile_factor (enum tree_code);
841
842
843 /* In tree-vect-slp.c. */
750 extern void vect_free_slp_instance (slp_instance); 844 extern void vect_free_slp_instance (slp_instance);
751 extern loop_vec_info vect_analyze_loop_form (struct loop *); 845 extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *,
752 extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *, 846 gimple_stmt_iterator *, int,
753 HOST_WIDE_INT *); 847 slp_instance, bool);
754 848 extern bool vect_schedule_slp (loop_vec_info, bb_vec_info);
755 /** In tree-vect-patterns.c **/ 849 extern void vect_update_slp_costs_according_to_vf (loop_vec_info);
850 extern bool vect_analyze_slp (loop_vec_info, bb_vec_info);
851 extern void vect_make_slp_decision (loop_vec_info);
852 extern void vect_detect_hybrid_slp (loop_vec_info);
853 extern void vect_get_slp_defs (slp_tree, VEC (tree,heap) **,
854 VEC (tree,heap) **);
855 extern LOC find_bb_location (basic_block);
856 extern bb_vec_info vect_slp_analyze_bb (basic_block);
857 extern void vect_slp_transform_bb (basic_block);
858
859 /* In tree-vect-patterns.c. */
756 /* Pattern recognition functions. 860 /* Pattern recognition functions.
757 Additional pattern recognition functions can (and will) be added 861 Additional pattern recognition functions can (and will) be added
758 in the future. */ 862 in the future. */
759 typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *); 863 typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *);
760 #define NUM_PATTERNS 4 864 #define NUM_PATTERNS 4
761 void vect_pattern_recog (loop_vec_info); 865 void vect_pattern_recog (loop_vec_info);
762 866
763 867 /* In tree-vectorizer.c. */
764 /** In tree-vect-transform.c **/ 868 unsigned vectorize_loops (void);
765 extern bool vectorizable_load (gimple, gimple_stmt_iterator *, gimple *, 869 /* Vectorization debug information */
766 slp_tree, slp_instance);
767 extern bool vectorizable_store (gimple, gimple_stmt_iterator *, gimple *,
768 slp_tree);
769 extern bool vectorizable_operation (gimple, gimple_stmt_iterator *, gimple *,
770 slp_tree);
771 extern bool vectorizable_type_promotion (gimple, gimple_stmt_iterator *,
772 gimple *, slp_tree);
773 extern bool vectorizable_type_demotion (gimple, gimple_stmt_iterator *,
774 gimple *, slp_tree);
775 extern bool vectorizable_conversion (gimple, gimple_stmt_iterator *, gimple *,
776 slp_tree);
777 extern bool vectorizable_assignment (gimple, gimple_stmt_iterator *, gimple *,
778 slp_tree);
779 extern tree vectorizable_function (gimple, tree, tree);
780 extern bool vectorizable_call (gimple, gimple_stmt_iterator *, gimple *);
781 extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *);
782 extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *,
783 gimple *);
784 extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *);
785 extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *);
786 extern int vect_estimate_min_profitable_iters (loop_vec_info);
787 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
788 slp_tree);
789 extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type,
790 slp_tree);
791 extern void vect_model_load_cost (stmt_vec_info, int, slp_tree);
792 extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *,
793 gimple_stmt_iterator *, int, slp_instance, bool);
794
795 /* Driver for transformation stage. */
796 extern void vect_transform_loop (loop_vec_info);
797
798 /*************************************************************************
799 Vectorization Debug Information - in tree-vectorizer.c
800 *************************************************************************/
801 extern bool vect_print_dump_info (enum verbosity_levels); 870 extern bool vect_print_dump_info (enum verbosity_levels);
802 extern void vect_set_verbosity_level (const char *);
803 extern LOC find_loop_location (struct loop *);
804 871
805 #endif /* GCC_TREE_VECTORIZER_H */ 872 #endif /* GCC_TREE_VECTORIZER_H */