comparison gcc/tree-if-conv.c @ 131:84e7813d76e9

gcc-8.2
author mir3636
date Thu, 25 Oct 2018 07:37:49 +0900
parents 04ced10e8804
children 1830386684a0
comparison
equal deleted inserted replaced
111:04ced10e8804 131:84e7813d76e9
1 /* If-conversion for vectorizer. 1 /* If-conversion for vectorizer.
2 Copyright (C) 2004-2017 Free Software Foundation, Inc. 2 Copyright (C) 2004-2018 Free Software Foundation, Inc.
3 Contributed by Devang Patel <dpatel@apple.com> 3 Contributed by Devang Patel <dpatel@apple.com>
4 4
5 This file is part of GCC. 5 This file is part of GCC.
6 6
7 GCC is free software; you can redistribute it and/or modify it under 7 GCC is free software; you can redistribute it and/or modify it under
114 #include "tree-hash-traits.h" 114 #include "tree-hash-traits.h"
115 #include "varasm.h" 115 #include "varasm.h"
116 #include "builtins.h" 116 #include "builtins.h"
117 #include "params.h" 117 #include "params.h"
118 #include "cfganal.h" 118 #include "cfganal.h"
119 #include "internal-fn.h"
120 #include "fold-const.h"
119 121
120 /* Only handle PHIs with no more arguments unless we are asked to by 122 /* Only handle PHIs with no more arguments unless we are asked to by
121 simd pragma. */ 123 simd pragma. */
122 #define MAX_PHI_ARG_NUM \ 124 #define MAX_PHI_ARG_NUM \
123 ((unsigned) PARAM_VALUE (PARAM_MAX_TREE_IF_CONVERSION_PHI_ARGS)) 125 ((unsigned) PARAM_VALUE (PARAM_MAX_TREE_IF_CONVERSION_PHI_ARGS))
124 126
125 /* Indicate if new load/store that needs to be predicated is introduced 127 /* True if we've converted a statement that was only executed when some
126 during if conversion. */ 128 condition C was true, and if for correctness we need to predicate the
127 static bool any_pred_load_store; 129 statement to ensure that it is a no-op when C is false. See
130 predicate_statements for the kinds of predication we support. */
131 static bool need_to_predicate;
128 132
129 /* Indicate if there are any complicated PHIs that need to be handled in 133 /* Indicate if there are any complicated PHIs that need to be handled in
130 if-conversion. Complicated PHI has more than two arguments and can't 134 if-conversion. Complicated PHI has more than two arguments and can't
131 be degenerated to two arguments PHI. See more information in comment 135 be degenerated to two arguments PHI. See more information in comment
132 before phi_convertible_by_degenerating_args. */ 136 before phi_convertible_by_degenerating_args. */
191 data_reference_p> *innermost_DR_map; 195 data_reference_p> *innermost_DR_map;
192 196
193 /* Hash table to store <base reference, DR> pairs. */ 197 /* Hash table to store <base reference, DR> pairs. */
194 static hash_map<tree_operand_hash, data_reference_p> *baseref_DR_map; 198 static hash_map<tree_operand_hash, data_reference_p> *baseref_DR_map;
195 199
200 /* List of redundant SSA names: the first should be replaced by the second. */
201 static vec< std::pair<tree, tree> > redundant_ssa_names;
202
196 /* Structure used to predicate basic blocks. This is attached to the 203 /* Structure used to predicate basic blocks. This is attached to the
197 ->aux field of the BBs in the loop to be if-converted. */ 204 ->aux field of the BBs in the loop to be if-converted. */
198 struct bb_predicate { 205 struct bb_predicate {
199 206
200 /* The condition under which this basic block is executed. */ 207 /* The condition under which this basic block is executed. */
255 of the predicate for basic block BB. */ 262 of the predicate for basic block BB. */
256 263
257 static inline void 264 static inline void
258 add_bb_predicate_gimplified_stmts (basic_block bb, gimple_seq stmts) 265 add_bb_predicate_gimplified_stmts (basic_block bb, gimple_seq stmts)
259 { 266 {
267 /* We might have updated some stmts in STMTS via force_gimple_operand
268 calling fold_stmt and that producing multiple stmts. Delink immediate
269 uses so update_ssa after loop versioning doesn't get confused for
270 the not yet inserted predicates.
271 ??? This should go away once we reliably avoid updating stmts
272 not in any BB. */
273 for (gimple_stmt_iterator gsi = gsi_start (stmts);
274 !gsi_end_p (gsi); gsi_next (&gsi))
275 {
276 gimple *stmt = gsi_stmt (gsi);
277 delink_stmt_imm_use (stmt);
278 gimple_set_modified (stmt, true);
279 }
260 gimple_seq_add_seq_without_update 280 gimple_seq_add_seq_without_update
261 (&(((struct bb_predicate *) bb->aux)->predicate_gimplified_stmts), stmts); 281 (&(((struct bb_predicate *) bb->aux)->predicate_gimplified_stmts), stmts);
262 } 282 }
263 283
264 /* Initializes to TRUE the predicate of basic block BB. */ 284 /* Initializes to TRUE the predicate of basic block BB. */
269 bb->aux = XNEW (struct bb_predicate); 289 bb->aux = XNEW (struct bb_predicate);
270 set_bb_predicate_gimplified_stmts (bb, NULL); 290 set_bb_predicate_gimplified_stmts (bb, NULL);
271 set_bb_predicate (bb, boolean_true_node); 291 set_bb_predicate (bb, boolean_true_node);
272 } 292 }
273 293
274 /* Release the SSA_NAMEs associated with the predicate of basic block BB, 294 /* Release the SSA_NAMEs associated with the predicate of basic block BB. */
275 but don't actually free it. */
276 295
277 static inline void 296 static inline void
278 release_bb_predicate (basic_block bb) 297 release_bb_predicate (basic_block bb)
279 { 298 {
280 gimple_seq stmts = bb_predicate_gimplified_stmts (bb); 299 gimple_seq stmts = bb_predicate_gimplified_stmts (bb);
281 if (stmts) 300 if (stmts)
282 { 301 {
302 /* Ensure that these stmts haven't yet been added to a bb. */
283 if (flag_checking) 303 if (flag_checking)
284 for (gimple_stmt_iterator i = gsi_start (stmts); 304 for (gimple_stmt_iterator i = gsi_start (stmts);
285 !gsi_end_p (i); gsi_next (&i)) 305 !gsi_end_p (i); gsi_next (&i))
286 gcc_assert (! gimple_use_ops (gsi_stmt (i))); 306 gcc_assert (! gimple_bb (gsi_stmt (i)));
287 307
308 /* Discard them. */
309 gimple_seq_discard (stmts);
288 set_bb_predicate_gimplified_stmts (bb, NULL); 310 set_bb_predicate_gimplified_stmts (bb, NULL);
289 } 311 }
290 } 312 }
291 313
292 /* Free the predicate of basic block BB. */ 314 /* Free the predicate of basic block BB. */
726 within array bound. Return false otherwise. */ 748 within array bound. Return false otherwise. */
727 749
728 static bool 750 static bool
729 idx_within_array_bound (tree ref, tree *idx, void *dta) 751 idx_within_array_bound (tree ref, tree *idx, void *dta)
730 { 752 {
731 bool overflow; 753 wi::overflow_type overflow;
732 widest_int niter, valid_niter, delta, wi_step; 754 widest_int niter, valid_niter, delta, wi_step;
733 tree ev, init, step; 755 tree ev, init, step;
734 tree low, high; 756 tree low, high;
735 struct loop *loop = (struct loop*) dta; 757 struct loop *loop = (struct loop*) dta;
736 758
847 which is defined as read and write and is bound to the definition 869 which is defined as read and write and is bound to the definition
848 we are seeing. */ 870 we are seeing. */
849 static bool 871 static bool
850 ifcvt_memrefs_wont_trap (gimple *stmt, vec<data_reference_p> drs) 872 ifcvt_memrefs_wont_trap (gimple *stmt, vec<data_reference_p> drs)
851 { 873 {
874 /* If DR didn't see a reference here we can't use it to tell
875 whether the ref traps or not. */
876 if (gimple_uid (stmt) == 0)
877 return false;
878
852 data_reference_p *master_dr, *base_master_dr; 879 data_reference_p *master_dr, *base_master_dr;
853 data_reference_p a = drs[gimple_uid (stmt) - 1]; 880 data_reference_p a = drs[gimple_uid (stmt) - 1];
854 881
855 tree base = DR_BASE_OBJECT (a); 882 tree base = DR_BASE_OBJECT (a);
856 innermost_loop_behavior *innermost = &DR_INNERMOST (a); 883 innermost_loop_behavior *innermost = &DR_INNERMOST (a);
897 (conditional load or store based on a mask computed from bb predicate). */ 924 (conditional load or store based on a mask computed from bb predicate). */
898 925
899 static bool 926 static bool
900 ifcvt_can_use_mask_load_store (gimple *stmt) 927 ifcvt_can_use_mask_load_store (gimple *stmt)
901 { 928 {
902 tree lhs, ref; 929 /* Check whether this is a load or store. */
903 machine_mode mode; 930 tree lhs = gimple_assign_lhs (stmt);
904 basic_block bb = gimple_bb (stmt);
905 bool is_load; 931 bool is_load;
906 932 tree ref;
907 if (!(flag_tree_loop_vectorize || bb->loop_father->force_vectorize)
908 || bb->loop_father->dont_vectorize
909 || !gimple_assign_single_p (stmt)
910 || gimple_has_volatile_ops (stmt))
911 return false;
912
913 /* Check whether this is a load or store. */
914 lhs = gimple_assign_lhs (stmt);
915 if (gimple_store_p (stmt)) 933 if (gimple_store_p (stmt))
916 { 934 {
917 if (!is_gimple_val (gimple_assign_rhs1 (stmt))) 935 if (!is_gimple_val (gimple_assign_rhs1 (stmt)))
918 return false; 936 return false;
919 is_load = false; 937 is_load = false;
930 if (may_be_nonaddressable_p (ref)) 948 if (may_be_nonaddressable_p (ref))
931 return false; 949 return false;
932 950
933 /* Mask should be integer mode of the same size as the load/store 951 /* Mask should be integer mode of the same size as the load/store
934 mode. */ 952 mode. */
935 mode = TYPE_MODE (TREE_TYPE (lhs)); 953 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
936 if (!int_mode_for_mode (mode).exists () || VECTOR_MODE_P (mode)) 954 if (!int_mode_for_mode (mode).exists () || VECTOR_MODE_P (mode))
937 return false; 955 return false;
938 956
939 if (can_vec_mask_load_store_p (mode, VOIDmode, is_load)) 957 if (can_vec_mask_load_store_p (mode, VOIDmode, is_load))
940 return true; 958 return true;
941 959
942 return false; 960 return false;
961 }
962
963 /* Return true if STMT could be converted from an operation that is
964 unconditional to one that is conditional on a bb predicate mask. */
965
966 static bool
967 ifcvt_can_predicate (gimple *stmt)
968 {
969 basic_block bb = gimple_bb (stmt);
970
971 if (!(flag_tree_loop_vectorize || bb->loop_father->force_vectorize)
972 || bb->loop_father->dont_vectorize
973 || gimple_has_volatile_ops (stmt))
974 return false;
975
976 if (gimple_assign_single_p (stmt))
977 return ifcvt_can_use_mask_load_store (stmt);
978
979 tree_code code = gimple_assign_rhs_code (stmt);
980 tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
981 tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
982 if (!types_compatible_p (lhs_type, rhs_type))
983 return false;
984 internal_fn cond_fn = get_conditional_internal_fn (code);
985 return (cond_fn != IFN_LAST
986 && vectorized_internal_fn_supported_p (cond_fn, lhs_type));
943 } 987 }
944 988
945 /* Return true when STMT is if-convertible. 989 /* Return true when STMT is if-convertible.
946 990
947 GIMPLE_ASSIGN statement is not if-convertible if, 991 GIMPLE_ASSIGN statement is not if-convertible if,
984 if ((! gimple_vuse (stmt) 1028 if ((! gimple_vuse (stmt)
985 || gimple_could_trap_p_1 (stmt, false, false) 1029 || gimple_could_trap_p_1 (stmt, false, false)
986 || ! ifcvt_memrefs_wont_trap (stmt, refs)) 1030 || ! ifcvt_memrefs_wont_trap (stmt, refs))
987 && gimple_could_trap_p (stmt)) 1031 && gimple_could_trap_p (stmt))
988 { 1032 {
989 if (ifcvt_can_use_mask_load_store (stmt)) 1033 if (ifcvt_can_predicate (stmt))
990 { 1034 {
991 gimple_set_plf (stmt, GF_PLF_2, true); 1035 gimple_set_plf (stmt, GF_PLF_2, true);
992 any_pred_load_store = true; 1036 need_to_predicate = true;
993 return true; 1037 return true;
994 } 1038 }
995 if (dump_file && (dump_flags & TDF_DETAILS)) 1039 if (dump_file && (dump_flags & TDF_DETAILS))
996 fprintf (dump_file, "tree could trap...\n"); 1040 fprintf (dump_file, "tree could trap...\n");
997 return false; 1041 return false;
998 } 1042 }
999 1043
1000 /* When if-converting stores force versioning, likewise if we 1044 /* When if-converting stores force versioning, likewise if we
1001 ended up generating store data races. */ 1045 ended up generating store data races. */
1002 if (gimple_vdef (stmt)) 1046 if (gimple_vdef (stmt))
1003 any_pred_load_store = true; 1047 need_to_predicate = true;
1004 1048
1005 return true; 1049 return true;
1006 } 1050 }
1007 1051
1008 /* Return true when STMT is if-convertible. 1052 /* Return true when STMT is if-convertible.
1033 int flags = gimple_call_flags (stmt); 1077 int flags = gimple_call_flags (stmt);
1034 if ((flags & ECF_CONST) 1078 if ((flags & ECF_CONST)
1035 && !(flags & ECF_LOOPING_CONST_OR_PURE) 1079 && !(flags & ECF_LOOPING_CONST_OR_PURE)
1036 /* We can only vectorize some builtins at the moment, 1080 /* We can only vectorize some builtins at the moment,
1037 so restrict if-conversion to those. */ 1081 so restrict if-conversion to those. */
1038 && DECL_BUILT_IN (fndecl)) 1082 && fndecl_built_in_p (fndecl))
1039 return true; 1083 return true;
1040 } 1084 }
1041 return false; 1085 return false;
1042 } 1086 }
1043 1087
1066 1110
1067 FOR_EACH_EDGE (e, ei, bb->preds) 1111 FOR_EACH_EDGE (e, ei, bb->preds)
1068 if (EDGE_COUNT (e->src->succs) == 1) 1112 if (EDGE_COUNT (e->src->succs) == 1)
1069 return false; 1113 return false;
1070 return true; 1114 return true;
1071 }
1072
1073 /* Returns true if at least one successor in on critical edge. */
1074 static inline bool
1075 has_pred_critical_p (basic_block bb)
1076 {
1077 edge e;
1078 edge_iterator ei;
1079
1080 FOR_EACH_EDGE (e, ei, bb->preds)
1081 if (EDGE_COUNT (e->src->succs) > 1)
1082 return true;
1083 return false;
1084 } 1115 }
1085 1116
1086 /* Return true when BB is if-convertible. This routine does not check 1117 /* Return true when BB is if-convertible. This routine does not check
1087 basic block's statements and phis. 1118 basic block's statements and phis.
1088 1119
2030 } 2061 }
2031 2062
2032 stmts = bb_predicate_gimplified_stmts (bb); 2063 stmts = bb_predicate_gimplified_stmts (bb);
2033 if (stmts) 2064 if (stmts)
2034 { 2065 {
2035 if (any_pred_load_store) 2066 if (need_to_predicate)
2036 { 2067 {
2037 /* Insert the predicate of the BB just after the label, 2068 /* Insert the predicate of the BB just after the label,
2038 as the if-conversion of memory writes will use this 2069 as the if-conversion of memory writes will use this
2039 predicate. */ 2070 predicate. */
2040 gimple_stmt_iterator gsi = gsi_after_labels (bb); 2071 gimple_stmt_iterator gsi = gsi_after_labels (bb);
2058 set_bb_predicate_gimplified_stmts (bb, NULL); 2089 set_bb_predicate_gimplified_stmts (bb, NULL);
2059 } 2090 }
2060 } 2091 }
2061 } 2092 }
2062 2093
2063 /* Helper function for predicate_mem_writes. Returns index of existent 2094 /* Helper function for predicate_statements. Returns index of existent
2064 mask if it was created for given SIZE and -1 otherwise. */ 2095 mask if it was created for given SIZE and -1 otherwise. */
2065 2096
2066 static int 2097 static int
2067 mask_exists (int size, vec<int> vec) 2098 mask_exists (int size, vec<int> vec)
2068 { 2099 {
2070 int v; 2101 int v;
2071 FOR_EACH_VEC_ELT (vec, ix, v) 2102 FOR_EACH_VEC_ELT (vec, ix, v)
2072 if (v == size) 2103 if (v == size)
2073 return (int) ix; 2104 return (int) ix;
2074 return -1; 2105 return -1;
2106 }
2107
2108 /* Helper function for predicate_statements. STMT is a memory read or
2109 write and it needs to be predicated by MASK. Return a statement
2110 that does so. */
2111
2112 static gimple *
2113 predicate_load_or_store (gimple_stmt_iterator *gsi, gassign *stmt, tree mask)
2114 {
2115 gcall *new_stmt;
2116
2117 tree lhs = gimple_assign_lhs (stmt);
2118 tree rhs = gimple_assign_rhs1 (stmt);
2119 tree ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs;
2120 mark_addressable (ref);
2121 tree addr = force_gimple_operand_gsi (gsi, build_fold_addr_expr (ref),
2122 true, NULL_TREE, true, GSI_SAME_STMT);
2123 tree ptr = build_int_cst (reference_alias_ptr_type (ref),
2124 get_object_alignment (ref));
2125 /* Copy points-to info if possible. */
2126 if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr))
2127 copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr),
2128 ref);
2129 if (TREE_CODE (lhs) == SSA_NAME)
2130 {
2131 new_stmt
2132 = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr,
2133 ptr, mask);
2134 gimple_call_set_lhs (new_stmt, lhs);
2135 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2136 }
2137 else
2138 {
2139 new_stmt
2140 = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr,
2141 mask, rhs);
2142 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2143 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
2144 SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
2145 }
2146 gimple_call_set_nothrow (new_stmt, true);
2147 return new_stmt;
2148 }
2149
2150 /* STMT uses OP_LHS. Check whether it is equivalent to:
2151
2152 ... = OP_MASK ? OP_LHS : X;
2153
2154 Return X if so, otherwise return null. OP_MASK is an SSA_NAME that is
2155 known to have value OP_COND. */
2156
2157 static tree
2158 check_redundant_cond_expr (gimple *stmt, tree op_mask, tree op_cond,
2159 tree op_lhs)
2160 {
2161 gassign *assign = dyn_cast <gassign *> (stmt);
2162 if (!assign || gimple_assign_rhs_code (assign) != COND_EXPR)
2163 return NULL_TREE;
2164
2165 tree use_cond = gimple_assign_rhs1 (assign);
2166 tree if_true = gimple_assign_rhs2 (assign);
2167 tree if_false = gimple_assign_rhs3 (assign);
2168
2169 if ((use_cond == op_mask || operand_equal_p (use_cond, op_cond, 0))
2170 && if_true == op_lhs)
2171 return if_false;
2172
2173 if (inverse_conditions_p (use_cond, op_cond) && if_false == op_lhs)
2174 return if_true;
2175
2176 return NULL_TREE;
2177 }
2178
2179 /* Return true if VALUE is available for use at STMT. SSA_NAMES is
2180 the set of SSA names defined earlier in STMT's block. */
2181
2182 static bool
2183 value_available_p (gimple *stmt, hash_set<tree_ssa_name_hash> *ssa_names,
2184 tree value)
2185 {
2186 if (is_gimple_min_invariant (value))
2187 return true;
2188
2189 if (TREE_CODE (value) == SSA_NAME)
2190 {
2191 if (SSA_NAME_IS_DEFAULT_DEF (value))
2192 return true;
2193
2194 basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (value));
2195 basic_block use_bb = gimple_bb (stmt);
2196 return (def_bb == use_bb
2197 ? ssa_names->contains (value)
2198 : dominated_by_p (CDI_DOMINATORS, use_bb, def_bb));
2199 }
2200
2201 return false;
2202 }
2203
2204 /* Helper function for predicate_statements. STMT is a potentially-trapping
2205 arithmetic operation that needs to be predicated by MASK, an SSA_NAME that
2206 has value COND. Return a statement that does so. SSA_NAMES is the set of
2207 SSA names defined earlier in STMT's block. */
2208
2209 static gimple *
2210 predicate_rhs_code (gassign *stmt, tree mask, tree cond,
2211 hash_set<tree_ssa_name_hash> *ssa_names)
2212 {
2213 tree lhs = gimple_assign_lhs (stmt);
2214 tree_code code = gimple_assign_rhs_code (stmt);
2215 unsigned int nops = gimple_num_ops (stmt);
2216 internal_fn cond_fn = get_conditional_internal_fn (code);
2217
2218 /* Construct the arguments to the conditional internal function. */
2219 auto_vec<tree, 8> args;
2220 args.safe_grow (nops + 1);
2221 args[0] = mask;
2222 for (unsigned int i = 1; i < nops; ++i)
2223 args[i] = gimple_op (stmt, i);
2224 args[nops] = NULL_TREE;
2225
2226 /* Look for uses of the result to see whether they are COND_EXPRs that can
2227 be folded into the conditional call. */
2228 imm_use_iterator imm_iter;
2229 gimple *use_stmt;
2230 FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
2231 {
2232 tree new_else = check_redundant_cond_expr (use_stmt, mask, cond, lhs);
2233 if (new_else && value_available_p (stmt, ssa_names, new_else))
2234 {
2235 if (!args[nops])
2236 args[nops] = new_else;
2237 if (operand_equal_p (new_else, args[nops], 0))
2238 {
2239 /* We have:
2240
2241 LHS = IFN_COND (MASK, ..., ELSE);
2242 X = MASK ? LHS : ELSE;
2243
2244 which makes X equivalent to LHS. */
2245 tree use_lhs = gimple_assign_lhs (use_stmt);
2246 redundant_ssa_names.safe_push (std::make_pair (use_lhs, lhs));
2247 }
2248 }
2249 }
2250 if (!args[nops])
2251 args[nops] = targetm.preferred_else_value (cond_fn, TREE_TYPE (lhs),
2252 nops - 1, &args[1]);
2253
2254 /* Create and insert the call. */
2255 gcall *new_stmt = gimple_build_call_internal_vec (cond_fn, args);
2256 gimple_call_set_lhs (new_stmt, lhs);
2257 gimple_call_set_nothrow (new_stmt, true);
2258
2259 return new_stmt;
2075 } 2260 }
2076 2261
2077 /* Predicate each write to memory in LOOP. 2262 /* Predicate each write to memory in LOOP.
2078 2263
2079 This function transforms control flow constructs containing memory 2264 This function transforms control flow constructs containing memory
2136 | 2321 |
2137 | bb_4 2322 | bb_4
2138 | goto bb_1 2323 | goto bb_1
2139 | end_bb_4 2324 | end_bb_4
2140 2325
2141 predicate_mem_writes is then predicating the memory write as follows: 2326 predicate_statements is then predicating the memory write as follows:
2142 2327
2143 | bb_0 2328 | bb_0
2144 | i = 0 2329 | i = 0
2145 | end_bb_0 2330 | end_bb_0
2146 | 2331 |
2180 | goto bb_1 2365 | goto bb_1
2181 | end_bb_4 2366 | end_bb_4
2182 */ 2367 */
2183 2368
2184 static void 2369 static void
2185 predicate_mem_writes (loop_p loop) 2370 predicate_statements (loop_p loop)
2186 { 2371 {
2187 unsigned int i, orig_loop_num_nodes = loop->num_nodes; 2372 unsigned int i, orig_loop_num_nodes = loop->num_nodes;
2188 auto_vec<int, 1> vect_sizes; 2373 auto_vec<int, 1> vect_sizes;
2189 auto_vec<tree, 1> vect_masks; 2374 auto_vec<tree, 1> vect_masks;
2375 hash_set<tree_ssa_name_hash> ssa_names;
2190 2376
2191 for (i = 1; i < orig_loop_num_nodes; i++) 2377 for (i = 1; i < orig_loop_num_nodes; i++)
2192 { 2378 {
2193 gimple_stmt_iterator gsi; 2379 gimple_stmt_iterator gsi;
2194 basic_block bb = ifc_bbs[i]; 2380 basic_block bb = ifc_bbs[i];
2195 tree cond = bb_predicate (bb); 2381 tree cond = bb_predicate (bb);
2196 bool swap; 2382 bool swap;
2197 gimple *stmt;
2198 int index; 2383 int index;
2199 2384
2200 if (is_true_predicate (cond)) 2385 if (is_true_predicate (cond))
2201 continue; 2386 continue;
2202 2387
2210 vect_sizes.truncate (0); 2395 vect_sizes.truncate (0);
2211 vect_masks.truncate (0); 2396 vect_masks.truncate (0);
2212 2397
2213 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) 2398 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
2214 { 2399 {
2215 if (!gimple_assign_single_p (stmt = gsi_stmt (gsi))) 2400 gassign *stmt = dyn_cast <gassign *> (gsi_stmt (gsi));
2401 if (!stmt)
2216 ; 2402 ;
2217 else if (is_false_predicate (cond) 2403 else if (is_false_predicate (cond)
2218 && gimple_vdef (stmt)) 2404 && gimple_vdef (stmt))
2219 { 2405 {
2220 unlink_stmt_vdef (stmt); 2406 unlink_stmt_vdef (stmt);
2223 continue; 2409 continue;
2224 } 2410 }
2225 else if (gimple_plf (stmt, GF_PLF_2)) 2411 else if (gimple_plf (stmt, GF_PLF_2))
2226 { 2412 {
2227 tree lhs = gimple_assign_lhs (stmt); 2413 tree lhs = gimple_assign_lhs (stmt);
2228 tree rhs = gimple_assign_rhs1 (stmt); 2414 tree mask;
2229 tree ref, addr, ptr, mask; 2415 gimple *new_stmt;
2230 gcall *new_stmt;
2231 gimple_seq stmts = NULL; 2416 gimple_seq stmts = NULL;
2232 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (lhs))); 2417 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
2233 ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs; 2418 /* We checked before setting GF_PLF_2 that an equivalent
2234 mark_addressable (ref); 2419 integer mode exists. */
2235 addr = force_gimple_operand_gsi (&gsi, build_fold_addr_expr (ref), 2420 int bitsize = GET_MODE_BITSIZE (mode).to_constant ();
2236 true, NULL_TREE, true,
2237 GSI_SAME_STMT);
2238 if (!vect_sizes.is_empty () 2421 if (!vect_sizes.is_empty ()
2239 && (index = mask_exists (bitsize, vect_sizes)) != -1) 2422 && (index = mask_exists (bitsize, vect_sizes)) != -1)
2240 /* Use created mask. */ 2423 /* Use created mask. */
2241 mask = vect_masks[index]; 2424 mask = vect_masks[index];
2242 else 2425 else
2245 mask = gimple_build (&stmts, TREE_CODE (cond), 2428 mask = gimple_build (&stmts, TREE_CODE (cond),
2246 boolean_type_node, 2429 boolean_type_node,
2247 TREE_OPERAND (cond, 0), 2430 TREE_OPERAND (cond, 0),
2248 TREE_OPERAND (cond, 1)); 2431 TREE_OPERAND (cond, 1));
2249 else 2432 else
2250 { 2433 mask = cond;
2251 gcc_assert (TREE_CODE (cond) == SSA_NAME);
2252 mask = cond;
2253 }
2254 2434
2255 if (swap) 2435 if (swap)
2256 { 2436 {
2257 tree true_val 2437 tree true_val
2258 = constant_boolean_node (true, TREE_TYPE (mask)); 2438 = constant_boolean_node (true, TREE_TYPE (mask));
2259 mask = gimple_build (&stmts, BIT_XOR_EXPR, 2439 mask = gimple_build (&stmts, BIT_XOR_EXPR,
2260 TREE_TYPE (mask), mask, true_val); 2440 TREE_TYPE (mask), mask, true_val);
2261 } 2441 }
2262 gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT); 2442 gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
2263 2443
2264 mask = ifc_temp_var (TREE_TYPE (mask), mask, &gsi);
2265 /* Save mask and its size for further use. */ 2444 /* Save mask and its size for further use. */
2266 vect_sizes.safe_push (bitsize); 2445 vect_sizes.safe_push (bitsize);
2267 vect_masks.safe_push (mask); 2446 vect_masks.safe_push (mask);
2268 } 2447 }
2269 ptr = build_int_cst (reference_alias_ptr_type (ref), 2448 if (gimple_assign_single_p (stmt))
2270 get_object_alignment (ref)); 2449 new_stmt = predicate_load_or_store (&gsi, stmt, mask);
2271 /* Copy points-to info if possible. */
2272 if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr))
2273 copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr),
2274 ref);
2275 if (TREE_CODE (lhs) == SSA_NAME)
2276 {
2277 new_stmt
2278 = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr,
2279 ptr, mask);
2280 gimple_call_set_lhs (new_stmt, lhs);
2281 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2282 }
2283 else 2450 else
2284 { 2451 new_stmt = predicate_rhs_code (stmt, mask, cond, &ssa_names);
2285 new_stmt
2286 = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr,
2287 mask, rhs);
2288 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2289 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
2290 SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
2291 }
2292 gimple_call_set_nothrow (new_stmt, true);
2293 2452
2294 gsi_replace (&gsi, new_stmt, true); 2453 gsi_replace (&gsi, new_stmt, true);
2295 } 2454 }
2296 else if (gimple_vdef (stmt)) 2455 else if (gimple_vdef (stmt))
2297 { 2456 {
2308 true, GSI_SAME_STMT); 2467 true, GSI_SAME_STMT);
2309 rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs); 2468 rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs);
2310 gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi)); 2469 gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi));
2311 update_stmt (stmt); 2470 update_stmt (stmt);
2312 } 2471 }
2472 tree lhs = gimple_get_lhs (gsi_stmt (gsi));
2473 if (lhs && TREE_CODE (lhs) == SSA_NAME)
2474 ssa_names.add (lhs);
2313 gsi_next (&gsi); 2475 gsi_next (&gsi);
2314 } 2476 }
2477 ssa_names.empty ();
2315 } 2478 }
2316 } 2479 }
2317 2480
2318 /* Remove all GIMPLE_CONDs and GIMPLE_LABELs of all the basic blocks 2481 /* Remove all GIMPLE_CONDs and GIMPLE_LABELs of all the basic blocks
2319 other than the exit and latch of the LOOP. Also resets the 2482 other than the exit and latch of the LOOP. Also resets the
2371 2534
2372 remove_conditions_and_labels (loop); 2535 remove_conditions_and_labels (loop);
2373 insert_gimplified_predicates (loop); 2536 insert_gimplified_predicates (loop);
2374 predicate_all_scalar_phis (loop); 2537 predicate_all_scalar_phis (loop);
2375 2538
2376 if (any_pred_load_store) 2539 if (need_to_predicate)
2377 predicate_mem_writes (loop); 2540 predicate_statements (loop);
2378 2541
2379 /* Merge basic blocks: first remove all the edges in the loop, 2542 /* Merge basic blocks: first remove all the edges in the loop,
2380 except for those from the exit block. */ 2543 except for those from the exit block. */
2381 exit_bb = NULL; 2544 exit_bb = NULL;
2382 bool *predicated = XNEWVEC (bool, orig_loop_num_nodes); 2545 bool *predicated = XNEWVEC (bool, orig_loop_num_nodes);
2712 gimple_stmt_iterator gsi; 2875 gimple_stmt_iterator gsi;
2713 auto_vec<gimple *> worklist; 2876 auto_vec<gimple *> worklist;
2714 enum gimple_code code; 2877 enum gimple_code code;
2715 use_operand_p use_p; 2878 use_operand_p use_p;
2716 imm_use_iterator imm_iter; 2879 imm_use_iterator imm_iter;
2880 std::pair <tree, tree> *name_pair;
2881 unsigned int i;
2882
2883 FOR_EACH_VEC_ELT (redundant_ssa_names, i, name_pair)
2884 replace_uses_by (name_pair->first, name_pair->second);
2885 redundant_ssa_names.release ();
2717 2886
2718 worklist.create (64); 2887 worklist.create (64);
2719 /* Consider all phi as live statements. */ 2888 /* Consider all phi as live statements. */
2720 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 2889 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2721 { 2890 {
2812 struct loop *rloop; 2981 struct loop *rloop;
2813 2982
2814 again: 2983 again:
2815 rloop = NULL; 2984 rloop = NULL;
2816 ifc_bbs = NULL; 2985 ifc_bbs = NULL;
2817 any_pred_load_store = false; 2986 need_to_predicate = false;
2818 any_complicated_phi = false; 2987 any_complicated_phi = false;
2819 2988
2820 /* Apply more aggressive if-conversion when loop or its outer loop were 2989 /* Apply more aggressive if-conversion when loop or its outer loop were
2821 marked with simd pragma. When that's the case, we try to if-convert 2990 marked with simd pragma. When that's the case, we try to if-convert
2822 loop containing PHIs with more than MAX_PHI_ARG_NUM arguments. */ 2991 loop containing PHIs with more than MAX_PHI_ARG_NUM arguments. */
2833 3002
2834 if (!if_convertible_loop_p (loop) 3003 if (!if_convertible_loop_p (loop)
2835 || !dbg_cnt (if_conversion_tree)) 3004 || !dbg_cnt (if_conversion_tree))
2836 goto cleanup; 3005 goto cleanup;
2837 3006
2838 if ((any_pred_load_store || any_complicated_phi) 3007 if ((need_to_predicate || any_complicated_phi)
2839 && ((!flag_tree_loop_vectorize && !loop->force_vectorize) 3008 && ((!flag_tree_loop_vectorize && !loop->force_vectorize)
2840 || loop->dont_vectorize)) 3009 || loop->dont_vectorize))
2841 goto cleanup; 3010 goto cleanup;
2842 3011
2843 /* Since we have no cost model, always version loops unless the user 3012 /* Since we have no cost model, always version loops unless the user
2844 specified -ftree-loop-if-convert or unless versioning is required. 3013 specified -ftree-loop-if-convert or unless versioning is required.
2845 Either version this loop, or if the pattern is right for outer-loop 3014 Either version this loop, or if the pattern is right for outer-loop
2846 vectorization, version the outer loop. In the latter case we will 3015 vectorization, version the outer loop. In the latter case we will
2847 still if-convert the original inner loop. */ 3016 still if-convert the original inner loop. */
2848 if (any_pred_load_store 3017 if (need_to_predicate
2849 || any_complicated_phi 3018 || any_complicated_phi
2850 || flag_tree_loop_if_convert != 1) 3019 || flag_tree_loop_if_convert != 1)
2851 { 3020 {
2852 struct loop *vloop 3021 struct loop *vloop
2853 = (versionable_outer_loop_p (loop_outer (loop)) 3022 = (versionable_outer_loop_p (loop_outer (loop))
2960 if (flag_tree_loop_if_convert == 1 3129 if (flag_tree_loop_if_convert == 1
2961 || ((flag_tree_loop_vectorize || loop->force_vectorize) 3130 || ((flag_tree_loop_vectorize || loop->force_vectorize)
2962 && !loop->dont_vectorize)) 3131 && !loop->dont_vectorize))
2963 todo |= tree_if_conversion (loop); 3132 todo |= tree_if_conversion (loop);
2964 3133
3134 if (todo)
3135 {
3136 free_numbers_of_iterations_estimates (fun);
3137 scev_reset ();
3138 }
3139
2965 if (flag_checking) 3140 if (flag_checking)
2966 { 3141 {
2967 basic_block bb; 3142 basic_block bb;
2968 FOR_EACH_BB_FN (bb, fun) 3143 FOR_EACH_BB_FN (bb, fun)
2969 gcc_assert (!bb->aux); 3144 gcc_assert (!bb->aux);