Mercurial > hg > CbC > CbC_gcc
comparison gcc/ipa-split.c @ 111:04ced10e8804
gcc 7
author | kono |
---|---|
date | Fri, 27 Oct 2017 22:46:09 +0900 |
parents | 561a7518be6b |
children | 84e7813d76e9 |
comparison
equal
deleted
inserted
replaced
68:561a7518be6b | 111:04ced10e8804 |
---|---|
1 /* Function splitting pass | 1 /* Function splitting pass |
2 Copyright (C) 2010, 2011 | 2 Copyright (C) 2010-2017 Free Software Foundation, Inc. |
3 Free Software Foundation, Inc. | |
4 Contributed by Jan Hubicka <jh@suse.cz> | 3 Contributed by Jan Hubicka <jh@suse.cz> |
5 | 4 |
6 This file is part of GCC. | 5 This file is part of GCC. |
7 | 6 |
8 GCC is free software; you can redistribute it and/or modify it under | 7 GCC is free software; you can redistribute it and/or modify it under |
76 when needed or splitting also the parts. */ | 75 when needed or splitting also the parts. */ |
77 | 76 |
78 #include "config.h" | 77 #include "config.h" |
79 #include "system.h" | 78 #include "system.h" |
80 #include "coretypes.h" | 79 #include "coretypes.h" |
80 #include "backend.h" | |
81 #include "rtl.h" | |
81 #include "tree.h" | 82 #include "tree.h" |
82 #include "target.h" | 83 #include "gimple.h" |
84 #include "cfghooks.h" | |
85 #include "alloc-pool.h" | |
86 #include "tree-pass.h" | |
87 #include "ssa.h" | |
83 #include "cgraph.h" | 88 #include "cgraph.h" |
89 #include "diagnostic.h" | |
90 #include "fold-const.h" | |
91 #include "cfganal.h" | |
92 #include "calls.h" | |
93 #include "gimplify.h" | |
94 #include "gimple-iterator.h" | |
95 #include "gimplify-me.h" | |
96 #include "gimple-walk.h" | |
97 #include "symbol-summary.h" | |
84 #include "ipa-prop.h" | 98 #include "ipa-prop.h" |
85 #include "tree-flow.h" | 99 #include "tree-cfg.h" |
86 #include "tree-pass.h" | 100 #include "tree-into-ssa.h" |
87 #include "flags.h" | 101 #include "tree-dfa.h" |
88 #include "timevar.h" | |
89 #include "diagnostic.h" | |
90 #include "tree-dump.h" | |
91 #include "tree-inline.h" | 102 #include "tree-inline.h" |
92 #include "fibheap.h" | |
93 #include "params.h" | 103 #include "params.h" |
94 #include "gimple-pretty-print.h" | 104 #include "gimple-pretty-print.h" |
105 #include "ipa-fnsummary.h" | |
106 #include "cfgloop.h" | |
107 #include "tree-chkp.h" | |
95 | 108 |
96 /* Per basic block info. */ | 109 /* Per basic block info. */ |
97 | 110 |
98 typedef struct | 111 struct split_bb_info |
99 { | 112 { |
100 unsigned int size; | 113 unsigned int size; |
101 unsigned int time; | 114 unsigned int time; |
102 } bb_info; | 115 }; |
103 DEF_VEC_O(bb_info); | 116 |
104 DEF_VEC_ALLOC_O(bb_info,heap); | 117 static vec<split_bb_info> bb_info_vec; |
105 | |
106 static VEC(bb_info, heap) *bb_info_vec; | |
107 | 118 |
108 /* Description of split point. */ | 119 /* Description of split point. */ |
109 | 120 |
110 struct split_point | 121 struct split_point |
111 { | 122 { |
128 | 139 |
129 /* Best split point found. */ | 140 /* Best split point found. */ |
130 | 141 |
131 struct split_point best_split_point; | 142 struct split_point best_split_point; |
132 | 143 |
144 /* Set of basic blocks that are not allowed to dominate a split point. */ | |
145 | |
146 static bitmap forbidden_dominators; | |
147 | |
133 static tree find_retval (basic_block return_bb); | 148 static tree find_retval (basic_block return_bb); |
149 static tree find_retbnd (basic_block return_bb); | |
134 | 150 |
135 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic | 151 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic |
136 variable, check it if it is present in bitmap passed via DATA. */ | 152 variable, check it if it is present in bitmap passed via DATA. */ |
137 | 153 |
138 static bool | 154 static bool |
139 test_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data) | 155 test_nonssa_use (gimple *, tree t, tree, void *data) |
140 { | 156 { |
141 t = get_base_address (t); | 157 t = get_base_address (t); |
142 | 158 |
143 if (!t || is_gimple_reg (t)) | 159 if (!t || is_gimple_reg (t)) |
144 return false; | 160 return false; |
145 | 161 |
146 if (TREE_CODE (t) == PARM_DECL | 162 if (TREE_CODE (t) == PARM_DECL |
147 || (TREE_CODE (t) == VAR_DECL | 163 || (VAR_P (t) |
148 && auto_var_in_fn_p (t, current_function_decl)) | 164 && auto_var_in_fn_p (t, current_function_decl)) |
149 || TREE_CODE (t) == RESULT_DECL | 165 || TREE_CODE (t) == RESULT_DECL |
150 || TREE_CODE (t) == LABEL_DECL) | 166 /* Normal labels are part of CFG and will be handled gratefuly. |
167 Forced labels however can be used directly by statements and | |
168 need to stay in one partition along with their uses. */ | |
169 || (TREE_CODE (t) == LABEL_DECL | |
170 && FORCED_LABEL (t))) | |
151 return bitmap_bit_p ((bitmap)data, DECL_UID (t)); | 171 return bitmap_bit_p ((bitmap)data, DECL_UID (t)); |
152 | 172 |
153 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want | 173 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want |
154 to pretend that the value pointed to is actual result decl. */ | 174 to pretend that the value pointed to is actual result decl. */ |
155 if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t)) | 175 if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t)) |
156 && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME | 176 && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME |
177 && SSA_NAME_VAR (TREE_OPERAND (t, 0)) | |
157 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL | 178 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL |
158 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 179 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
159 return | 180 return |
160 bitmap_bit_p ((bitmap)data, | 181 bitmap_bit_p ((bitmap)data, |
161 DECL_UID (DECL_RESULT (current_function_decl))); | 182 DECL_UID (DECL_RESULT (current_function_decl))); |
167 | 188 |
168 static void | 189 static void |
169 dump_split_point (FILE * file, struct split_point *current) | 190 dump_split_point (FILE * file, struct split_point *current) |
170 { | 191 { |
171 fprintf (file, | 192 fprintf (file, |
172 "Split point at BB %i header time:%i header size: %i" | 193 "Split point at BB %i\n" |
173 " split time: %i split size: %i\n bbs: ", | 194 " header time: %i header size: %i\n" |
195 " split time: %i split size: %i\n bbs: ", | |
174 current->entry_bb->index, current->header_time, | 196 current->entry_bb->index, current->header_time, |
175 current->header_size, current->split_time, current->split_size); | 197 current->header_size, current->split_time, current->split_size); |
176 dump_bitmap (file, current->split_bbs); | 198 dump_bitmap (file, current->split_bbs); |
177 fprintf (file, " SSA names to pass: "); | 199 fprintf (file, " SSA names to pass: "); |
178 dump_bitmap (file, current->ssa_names_to_pass); | 200 dump_bitmap (file, current->ssa_names_to_pass); |
185 static bool | 207 static bool |
186 verify_non_ssa_vars (struct split_point *current, bitmap non_ssa_vars, | 208 verify_non_ssa_vars (struct split_point *current, bitmap non_ssa_vars, |
187 basic_block return_bb) | 209 basic_block return_bb) |
188 { | 210 { |
189 bitmap seen = BITMAP_ALLOC (NULL); | 211 bitmap seen = BITMAP_ALLOC (NULL); |
190 VEC (basic_block,heap) *worklist = NULL; | 212 vec<basic_block> worklist = vNULL; |
191 edge e; | 213 edge e; |
192 edge_iterator ei; | 214 edge_iterator ei; |
193 bool ok = true; | 215 bool ok = true; |
216 basic_block bb; | |
194 | 217 |
195 FOR_EACH_EDGE (e, ei, current->entry_bb->preds) | 218 FOR_EACH_EDGE (e, ei, current->entry_bb->preds) |
196 if (e->src != ENTRY_BLOCK_PTR | 219 if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun) |
197 && !bitmap_bit_p (current->split_bbs, e->src->index)) | 220 && !bitmap_bit_p (current->split_bbs, e->src->index)) |
198 { | 221 { |
199 VEC_safe_push (basic_block, heap, worklist, e->src); | 222 worklist.safe_push (e->src); |
200 bitmap_set_bit (seen, e->src->index); | 223 bitmap_set_bit (seen, e->src->index); |
201 } | 224 } |
202 | 225 |
203 while (!VEC_empty (basic_block, worklist)) | 226 while (!worklist.is_empty ()) |
204 { | 227 { |
205 gimple_stmt_iterator bsi; | 228 bb = worklist.pop (); |
206 basic_block bb = VEC_pop (basic_block, worklist); | |
207 | |
208 FOR_EACH_EDGE (e, ei, bb->preds) | 229 FOR_EACH_EDGE (e, ei, bb->preds) |
209 if (e->src != ENTRY_BLOCK_PTR | 230 if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun) |
210 && bitmap_set_bit (seen, e->src->index)) | 231 && bitmap_set_bit (seen, e->src->index)) |
211 { | 232 { |
212 gcc_checking_assert (!bitmap_bit_p (current->split_bbs, | 233 gcc_checking_assert (!bitmap_bit_p (current->split_bbs, |
213 e->src->index)); | 234 e->src->index)); |
214 VEC_safe_push (basic_block, heap, worklist, e->src); | 235 worklist.safe_push (e->src); |
215 } | 236 } |
216 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 237 for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi); |
217 { | 238 gsi_next (&bsi)) |
218 gimple stmt = gsi_stmt (bsi); | 239 { |
240 gimple *stmt = gsi_stmt (bsi); | |
219 if (is_gimple_debug (stmt)) | 241 if (is_gimple_debug (stmt)) |
220 continue; | 242 continue; |
221 if (walk_stmt_load_store_addr_ops | 243 if (walk_stmt_load_store_addr_ops |
222 (stmt, non_ssa_vars, test_nonssa_use, test_nonssa_use, | 244 (stmt, non_ssa_vars, test_nonssa_use, test_nonssa_use, |
223 test_nonssa_use)) | 245 test_nonssa_use)) |
224 { | 246 { |
225 ok = false; | 247 ok = false; |
226 goto done; | 248 goto done; |
227 } | 249 } |
228 if (gimple_code (stmt) == GIMPLE_LABEL | 250 if (glabel *label_stmt = dyn_cast <glabel *> (stmt)) |
229 && test_nonssa_use (stmt, gimple_label_label (stmt), | 251 if (test_nonssa_use (stmt, gimple_label_label (label_stmt), |
230 non_ssa_vars)) | 252 NULL_TREE, non_ssa_vars)) |
231 { | 253 { |
232 ok = false; | 254 ok = false; |
233 goto done; | 255 goto done; |
234 } | 256 } |
235 } | 257 } |
236 for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 258 for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi); |
259 gsi_next (&bsi)) | |
237 { | 260 { |
238 if (walk_stmt_load_store_addr_ops | 261 if (walk_stmt_load_store_addr_ops |
239 (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use, | 262 (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use, |
240 test_nonssa_use)) | 263 test_nonssa_use)) |
241 { | 264 { |
245 } | 268 } |
246 FOR_EACH_EDGE (e, ei, bb->succs) | 269 FOR_EACH_EDGE (e, ei, bb->succs) |
247 { | 270 { |
248 if (e->dest != return_bb) | 271 if (e->dest != return_bb) |
249 continue; | 272 continue; |
250 for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi); | 273 for (gphi_iterator bsi = gsi_start_phis (return_bb); |
274 !gsi_end_p (bsi); | |
251 gsi_next (&bsi)) | 275 gsi_next (&bsi)) |
252 { | 276 { |
253 gimple stmt = gsi_stmt (bsi); | 277 gphi *stmt = bsi.phi (); |
254 tree op = gimple_phi_arg_def (stmt, e->dest_idx); | 278 tree op = gimple_phi_arg_def (stmt, e->dest_idx); |
255 | 279 |
256 if (!is_gimple_reg (gimple_phi_result (stmt))) | 280 if (virtual_operand_p (gimple_phi_result (stmt))) |
257 continue; | 281 continue; |
258 if (TREE_CODE (op) != SSA_NAME | 282 if (TREE_CODE (op) != SSA_NAME |
259 && test_nonssa_use (stmt, op, non_ssa_vars)) | 283 && test_nonssa_use (stmt, op, op, non_ssa_vars)) |
260 { | 284 { |
261 ok = false; | 285 ok = false; |
262 goto done; | 286 goto done; |
263 } | 287 } |
264 } | 288 } |
265 } | 289 } |
266 } | 290 } |
291 | |
292 /* Verify that the rest of function does not define any label | |
293 used by the split part. */ | |
294 FOR_EACH_BB_FN (bb, cfun) | |
295 if (!bitmap_bit_p (current->split_bbs, bb->index) | |
296 && !bitmap_bit_p (seen, bb->index)) | |
297 { | |
298 gimple_stmt_iterator bsi; | |
299 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | |
300 if (glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (bsi))) | |
301 { | |
302 if (test_nonssa_use (label_stmt, | |
303 gimple_label_label (label_stmt), | |
304 NULL_TREE, non_ssa_vars)) | |
305 { | |
306 ok = false; | |
307 goto done; | |
308 } | |
309 } | |
310 else | |
311 break; | |
312 } | |
313 | |
267 done: | 314 done: |
268 BITMAP_FREE (seen); | 315 BITMAP_FREE (seen); |
269 VEC_free (basic_block, heap, worklist); | 316 worklist.release (); |
270 return ok; | 317 return ok; |
318 } | |
319 | |
320 /* If STMT is a call, check the callee against a list of forbidden | |
321 predicate functions. If a match is found, look for uses of the | |
322 call result in condition statements that compare against zero. | |
323 For each such use, find the block targeted by the condition | |
324 statement for the nonzero result, and set the bit for this block | |
325 in the forbidden dominators bitmap. The purpose of this is to avoid | |
326 selecting a split point where we are likely to lose the chance | |
327 to optimize away an unused function call. */ | |
328 | |
329 static void | |
330 check_forbidden_calls (gimple *stmt) | |
331 { | |
332 imm_use_iterator use_iter; | |
333 use_operand_p use_p; | |
334 tree lhs; | |
335 | |
336 /* At the moment, __builtin_constant_p is the only forbidden | |
337 predicate function call (see PR49642). */ | |
338 if (!gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)) | |
339 return; | |
340 | |
341 lhs = gimple_call_lhs (stmt); | |
342 | |
343 if (!lhs || TREE_CODE (lhs) != SSA_NAME) | |
344 return; | |
345 | |
346 FOR_EACH_IMM_USE_FAST (use_p, use_iter, lhs) | |
347 { | |
348 tree op1; | |
349 basic_block use_bb, forbidden_bb; | |
350 enum tree_code code; | |
351 edge true_edge, false_edge; | |
352 gcond *use_stmt; | |
353 | |
354 use_stmt = dyn_cast <gcond *> (USE_STMT (use_p)); | |
355 if (!use_stmt) | |
356 continue; | |
357 | |
358 /* Assuming canonical form for GIMPLE_COND here, with constant | |
359 in second position. */ | |
360 op1 = gimple_cond_rhs (use_stmt); | |
361 code = gimple_cond_code (use_stmt); | |
362 use_bb = gimple_bb (use_stmt); | |
363 | |
364 extract_true_false_edges_from_block (use_bb, &true_edge, &false_edge); | |
365 | |
366 /* We're only interested in comparisons that distinguish | |
367 unambiguously from zero. */ | |
368 if (!integer_zerop (op1) || code == LE_EXPR || code == GE_EXPR) | |
369 continue; | |
370 | |
371 if (code == EQ_EXPR) | |
372 forbidden_bb = false_edge->dest; | |
373 else | |
374 forbidden_bb = true_edge->dest; | |
375 | |
376 bitmap_set_bit (forbidden_dominators, forbidden_bb->index); | |
377 } | |
378 } | |
379 | |
380 /* If BB is dominated by any block in the forbidden dominators set, | |
381 return TRUE; else FALSE. */ | |
382 | |
383 static bool | |
384 dominated_by_forbidden (basic_block bb) | |
385 { | |
386 unsigned dom_bb; | |
387 bitmap_iterator bi; | |
388 | |
389 EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators, 1, dom_bb, bi) | |
390 { | |
391 if (dominated_by_p (CDI_DOMINATORS, bb, | |
392 BASIC_BLOCK_FOR_FN (cfun, dom_bb))) | |
393 return true; | |
394 } | |
395 | |
396 return false; | |
397 } | |
398 | |
399 /* For give split point CURRENT and return block RETURN_BB return 1 | |
400 if ssa name VAL is set by split part and 0 otherwise. */ | |
401 static bool | |
402 split_part_set_ssa_name_p (tree val, struct split_point *current, | |
403 basic_block return_bb) | |
404 { | |
405 if (TREE_CODE (val) != SSA_NAME) | |
406 return false; | |
407 | |
408 return (!SSA_NAME_IS_DEFAULT_DEF (val) | |
409 && (bitmap_bit_p (current->split_bbs, | |
410 gimple_bb (SSA_NAME_DEF_STMT (val))->index) | |
411 || gimple_bb (SSA_NAME_DEF_STMT (val)) == return_bb)); | |
271 } | 412 } |
272 | 413 |
273 /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa | 414 /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa |
274 variables used and RETURN_BB is return basic block. | 415 variables used and RETURN_BB is return basic block. |
275 See if we can split function here. */ | 416 See if we can split function here. */ |
281 tree parm; | 422 tree parm; |
282 unsigned int num_args = 0; | 423 unsigned int num_args = 0; |
283 unsigned int call_overhead; | 424 unsigned int call_overhead; |
284 edge e; | 425 edge e; |
285 edge_iterator ei; | 426 edge_iterator ei; |
286 gimple_stmt_iterator bsi; | 427 gphi_iterator bsi; |
287 unsigned int i; | 428 unsigned int i; |
288 int incoming_freq = 0; | 429 int incoming_freq = 0; |
289 tree retval; | 430 tree retval; |
431 tree retbnd; | |
432 bool back_edge = false; | |
290 | 433 |
291 if (dump_file && (dump_flags & TDF_DETAILS)) | 434 if (dump_file && (dump_flags & TDF_DETAILS)) |
292 dump_split_point (dump_file, current); | 435 dump_split_point (dump_file, current); |
293 | 436 |
294 FOR_EACH_EDGE (e, ei, current->entry_bb->preds) | 437 FOR_EACH_EDGE (e, ei, current->entry_bb->preds) |
295 if (!bitmap_bit_p (current->split_bbs, e->src->index)) | 438 { |
296 incoming_freq += EDGE_FREQUENCY (e); | 439 if (e->flags & EDGE_DFS_BACK) |
440 back_edge = true; | |
441 if (!bitmap_bit_p (current->split_bbs, e->src->index)) | |
442 incoming_freq += EDGE_FREQUENCY (e); | |
443 } | |
297 | 444 |
298 /* Do not split when we would end up calling function anyway. */ | 445 /* Do not split when we would end up calling function anyway. */ |
299 if (incoming_freq | 446 if (incoming_freq |
300 >= (ENTRY_BLOCK_PTR->frequency | 447 >= (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency |
301 * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100)) | 448 * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100)) |
302 { | 449 { |
303 if (dump_file && (dump_flags & TDF_DETAILS)) | 450 /* When profile is guessed, we can not expect it to give us |
304 fprintf (dump_file, | 451 realistic estimate on likelyness of function taking the |
305 " Refused: incoming frequency is too large.\n"); | 452 complex path. As a special case, when tail of the function is |
306 return; | 453 a loop, enable splitting since inlining code skipping the loop |
454 is likely noticeable win. */ | |
455 if (back_edge | |
456 && profile_status_for_fn (cfun) != PROFILE_READ | |
457 && incoming_freq < ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency) | |
458 { | |
459 if (dump_file && (dump_flags & TDF_DETAILS)) | |
460 fprintf (dump_file, | |
461 " Split before loop, accepting despite low frequencies %i %i.\n", | |
462 incoming_freq, | |
463 ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency); | |
464 } | |
465 else | |
466 { | |
467 if (dump_file && (dump_flags & TDF_DETAILS)) | |
468 fprintf (dump_file, | |
469 " Refused: incoming frequency is too large.\n"); | |
470 return; | |
471 } | |
307 } | 472 } |
308 | 473 |
309 if (!current->header_size) | 474 if (!current->header_size) |
310 { | 475 { |
311 if (dump_file && (dump_flags & TDF_DETAILS)) | 476 if (dump_file && (dump_flags & TDF_DETAILS)) |
315 | 480 |
316 /* Verify that PHI args on entry are either virtual or all their operands | 481 /* Verify that PHI args on entry are either virtual or all their operands |
317 incoming from header are the same. */ | 482 incoming from header are the same. */ |
318 for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 483 for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi)) |
319 { | 484 { |
320 gimple stmt = gsi_stmt (bsi); | 485 gphi *stmt = bsi.phi (); |
321 tree val = NULL; | 486 tree val = NULL; |
322 | 487 |
323 if (!is_gimple_reg (gimple_phi_result (stmt))) | 488 if (virtual_operand_p (gimple_phi_result (stmt))) |
324 continue; | 489 continue; |
325 for (i = 0; i < gimple_phi_num_args (stmt); i++) | 490 for (i = 0; i < gimple_phi_num_args (stmt); i++) |
326 { | 491 { |
327 edge e = gimple_phi_arg_edge (stmt, i); | 492 edge e = gimple_phi_arg_edge (stmt, i); |
328 if (!bitmap_bit_p (current->split_bbs, e->src->index)) | 493 if (!bitmap_bit_p (current->split_bbs, e->src->index)) |
355 fprintf (dump_file, | 520 fprintf (dump_file, |
356 " Refused: need to pass non-ssa param values\n"); | 521 " Refused: need to pass non-ssa param values\n"); |
357 return; | 522 return; |
358 } | 523 } |
359 } | 524 } |
360 else if (gimple_default_def (cfun, parm) | 525 else |
361 && bitmap_bit_p (current->ssa_names_to_pass, | 526 { |
362 SSA_NAME_VERSION (gimple_default_def | 527 tree ddef = ssa_default_def (cfun, parm); |
363 (cfun, parm)))) | 528 if (ddef |
364 { | 529 && bitmap_bit_p (current->ssa_names_to_pass, |
365 if (!VOID_TYPE_P (TREE_TYPE (parm))) | 530 SSA_NAME_VERSION (ddef))) |
366 call_overhead += estimate_move_cost (TREE_TYPE (parm)); | 531 { |
367 num_args++; | 532 if (!VOID_TYPE_P (TREE_TYPE (parm))) |
533 call_overhead += estimate_move_cost (TREE_TYPE (parm), false); | |
534 num_args++; | |
535 } | |
368 } | 536 } |
369 } | 537 } |
370 if (!VOID_TYPE_P (TREE_TYPE (current_function_decl))) | 538 if (!VOID_TYPE_P (TREE_TYPE (current_function_decl))) |
371 call_overhead += estimate_move_cost (TREE_TYPE (current_function_decl)); | 539 call_overhead += estimate_move_cost (TREE_TYPE (current_function_decl), |
540 false); | |
372 | 541 |
373 if (current->split_size <= call_overhead) | 542 if (current->split_size <= call_overhead) |
374 { | 543 { |
375 if (dump_file && (dump_flags & TDF_DETAILS)) | 544 if (dump_file && (dump_flags & TDF_DETAILS)) |
376 fprintf (dump_file, | 545 fprintf (dump_file, |
383 : MAX_INLINE_INSNS_AUTO)) | 552 : MAX_INLINE_INSNS_AUTO)) |
384 { | 553 { |
385 if (dump_file && (dump_flags & TDF_DETAILS)) | 554 if (dump_file && (dump_flags & TDF_DETAILS)) |
386 fprintf (dump_file, | 555 fprintf (dump_file, |
387 " Refused: header size is too large for inline candidate\n"); | 556 " Refused: header size is too large for inline candidate\n"); |
557 return; | |
558 } | |
559 | |
560 /* Splitting functions brings the target out of comdat group; this will | |
561 lead to code duplication if the function is reused by other unit. | |
562 Limit this duplication. This is consistent with limit in tree-sra.c | |
563 FIXME: with LTO we ought to be able to do better! */ | |
564 if (DECL_ONE_ONLY (current_function_decl) | |
565 && current->split_size >= (unsigned int) MAX_INLINE_INSNS_AUTO) | |
566 { | |
567 if (dump_file && (dump_flags & TDF_DETAILS)) | |
568 fprintf (dump_file, | |
569 " Refused: function is COMDAT and tail is too large\n"); | |
570 return; | |
571 } | |
572 /* For comdat functions also reject very small tails; those will likely get | |
573 inlined back and we do not want to risk the duplication overhead. | |
574 FIXME: with LTO we ought to be able to do better! */ | |
575 if (DECL_ONE_ONLY (current_function_decl) | |
576 && current->split_size | |
577 <= (unsigned int) PARAM_VALUE (PARAM_EARLY_INLINING_INSNS) / 2) | |
578 { | |
579 if (dump_file && (dump_flags & TDF_DETAILS)) | |
580 fprintf (dump_file, | |
581 " Refused: function is COMDAT and tail is too small\n"); | |
388 return; | 582 return; |
389 } | 583 } |
390 | 584 |
391 /* FIXME: we currently can pass only SSA function parameters to the split | 585 /* FIXME: we currently can pass only SSA function parameters to the split |
392 arguments. Once parm_adjustment infrastructure is supported by cloning, | 586 arguments. Once parm_adjustment infrastructure is supported by cloning, |
409 if (dump_file && (dump_flags & TDF_DETAILS)) | 603 if (dump_file && (dump_flags & TDF_DETAILS)) |
410 fprintf (dump_file, | 604 fprintf (dump_file, |
411 " Refused: split part has non-ssa uses\n"); | 605 " Refused: split part has non-ssa uses\n"); |
412 return; | 606 return; |
413 } | 607 } |
608 | |
609 /* If the split point is dominated by a forbidden block, reject | |
610 the split. */ | |
611 if (!bitmap_empty_p (forbidden_dominators) | |
612 && dominated_by_forbidden (current->entry_bb)) | |
613 { | |
614 if (dump_file && (dump_flags & TDF_DETAILS)) | |
615 fprintf (dump_file, | |
616 " Refused: split point dominated by forbidden block\n"); | |
617 return; | |
618 } | |
619 | |
414 /* See if retval used by return bb is computed by header or split part. | 620 /* See if retval used by return bb is computed by header or split part. |
415 When it is computed by split part, we need to produce return statement | 621 When it is computed by split part, we need to produce return statement |
416 in the split part and add code to header to pass it around. | 622 in the split part and add code to header to pass it around. |
417 | 623 |
418 This is bit tricky to test: | 624 This is bit tricky to test: |
421 2) Invariants are always computed by caller. | 627 2) Invariants are always computed by caller. |
422 3) For SSA we need to look if defining statement is in header or split part | 628 3) For SSA we need to look if defining statement is in header or split part |
423 4) For non-SSA we need to look where the var is computed. */ | 629 4) For non-SSA we need to look where the var is computed. */ |
424 retval = find_retval (return_bb); | 630 retval = find_retval (return_bb); |
425 if (!retval) | 631 if (!retval) |
426 current->split_part_set_retval = true; | 632 { |
633 /* If there is a return_bb with no return value in function returning | |
634 value by reference, also make the split part return void, otherwise | |
635 we expansion would try to create a non-POD temporary, which is | |
636 invalid. */ | |
637 if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun) | |
638 && DECL_RESULT (current_function_decl) | |
639 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | |
640 current->split_part_set_retval = false; | |
641 else | |
642 current->split_part_set_retval = true; | |
643 } | |
427 else if (is_gimple_min_invariant (retval)) | 644 else if (is_gimple_min_invariant (retval)) |
428 current->split_part_set_retval = false; | 645 current->split_part_set_retval = false; |
429 /* Special case is value returned by reference we record as if it was non-ssa | 646 /* Special case is value returned by reference we record as if it was non-ssa |
430 set to result_decl. */ | 647 set to result_decl. */ |
431 else if (TREE_CODE (retval) == SSA_NAME | 648 else if (TREE_CODE (retval) == SSA_NAME |
649 && SSA_NAME_VAR (retval) | |
432 && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL | 650 && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL |
433 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 651 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
434 current->split_part_set_retval | 652 current->split_part_set_retval |
435 = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval))); | 653 = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval))); |
436 else if (TREE_CODE (retval) == SSA_NAME) | 654 else if (TREE_CODE (retval) == SSA_NAME) |
437 current->split_part_set_retval | 655 current->split_part_set_retval |
438 = (!SSA_NAME_IS_DEFAULT_DEF (retval) | 656 = split_part_set_ssa_name_p (retval, current, return_bb); |
439 && (bitmap_bit_p (current->split_bbs, | |
440 gimple_bb (SSA_NAME_DEF_STMT (retval))->index) | |
441 || gimple_bb (SSA_NAME_DEF_STMT (retval)) == return_bb)); | |
442 else if (TREE_CODE (retval) == PARM_DECL) | 657 else if (TREE_CODE (retval) == PARM_DECL) |
443 current->split_part_set_retval = false; | 658 current->split_part_set_retval = false; |
444 else if (TREE_CODE (retval) == VAR_DECL | 659 else if (VAR_P (retval) |
445 || TREE_CODE (retval) == RESULT_DECL) | 660 || TREE_CODE (retval) == RESULT_DECL) |
446 current->split_part_set_retval | 661 current->split_part_set_retval |
447 = bitmap_bit_p (non_ssa_vars, DECL_UID (retval)); | 662 = bitmap_bit_p (non_ssa_vars, DECL_UID (retval)); |
448 else | 663 else |
449 current->split_part_set_retval = true; | 664 current->split_part_set_retval = true; |
450 | 665 |
666 /* See if retbnd used by return bb is computed by header or split part. */ | |
667 retbnd = find_retbnd (return_bb); | |
668 if (retbnd) | |
669 { | |
670 bool split_part_set_retbnd | |
671 = split_part_set_ssa_name_p (retbnd, current, return_bb); | |
672 | |
673 /* If we have both return value and bounds then keep their definitions | |
674 in a single function. We use SSA names to link returned bounds and | |
675 value and therefore do not handle cases when result is passed by | |
676 reference (which should not be our case anyway since bounds are | |
677 returned for pointers only). */ | |
678 if ((DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)) | |
679 && current->split_part_set_retval) | |
680 || split_part_set_retbnd != current->split_part_set_retval) | |
681 { | |
682 if (dump_file && (dump_flags & TDF_DETAILS)) | |
683 fprintf (dump_file, | |
684 " Refused: split point splits return value and bounds\n"); | |
685 return; | |
686 } | |
687 } | |
688 | |
451 /* split_function fixes up at most one PHI non-virtual PHI node in return_bb, | 689 /* split_function fixes up at most one PHI non-virtual PHI node in return_bb, |
452 for the return value. If there are other PHIs, give up. */ | 690 for the return value. If there are other PHIs, give up. */ |
453 if (return_bb != EXIT_BLOCK_PTR) | 691 if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
454 { | 692 { |
455 gimple_stmt_iterator psi; | 693 gphi_iterator psi; |
456 | 694 |
457 for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi)) | 695 for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi)) |
458 if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi))) | 696 if (!virtual_operand_p (gimple_phi_result (psi.phi ())) |
459 && !(retval | 697 && !(retval |
460 && current->split_part_set_retval | 698 && current->split_part_set_retval |
461 && TREE_CODE (retval) == SSA_NAME | 699 && TREE_CODE (retval) == SSA_NAME |
462 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)) | 700 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)) |
463 && SSA_NAME_DEF_STMT (retval) == gsi_stmt (psi))) | 701 && SSA_NAME_DEF_STMT (retval) == psi.phi ())) |
464 { | 702 { |
465 if (dump_file && (dump_flags & TDF_DETAILS)) | 703 if (dump_file && (dump_flags & TDF_DETAILS)) |
466 fprintf (dump_file, | 704 fprintf (dump_file, |
467 " Refused: return bb has extra PHIs\n"); | 705 " Refused: return bb has extra PHIs\n"); |
468 return; | 706 return; |
499 | 737 |
500 /* Return basic block containing RETURN statement. We allow basic blocks | 738 /* Return basic block containing RETURN statement. We allow basic blocks |
501 of the form: | 739 of the form: |
502 <retval> = tmp_var; | 740 <retval> = tmp_var; |
503 return <retval> | 741 return <retval> |
504 but return_bb can not be more complex than this. | 742 but return_bb can not be more complex than this (except for |
505 If nothing is found, return EXIT_BLOCK_PTR. | 743 -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there). |
744 If nothing is found, return the exit block. | |
506 | 745 |
507 When there are multiple RETURN statement, chose one with return value, | 746 When there are multiple RETURN statement, chose one with return value, |
508 since that one is more likely shared by multiple code paths. | 747 since that one is more likely shared by multiple code paths. |
509 | 748 |
510 Return BB is special, because for function splitting it is the only | 749 Return BB is special, because for function splitting it is the only |
515 | 754 |
516 static basic_block | 755 static basic_block |
517 find_return_bb (void) | 756 find_return_bb (void) |
518 { | 757 { |
519 edge e; | 758 edge e; |
520 basic_block return_bb = EXIT_BLOCK_PTR; | 759 basic_block return_bb = EXIT_BLOCK_PTR_FOR_FN (cfun); |
521 gimple_stmt_iterator bsi; | 760 gimple_stmt_iterator bsi; |
522 bool found_return = false; | 761 bool found_return = false; |
523 tree retval = NULL_TREE; | 762 tree retval = NULL_TREE; |
524 | 763 |
525 if (!single_pred_p (EXIT_BLOCK_PTR)) | 764 if (!single_pred_p (EXIT_BLOCK_PTR_FOR_FN (cfun))) |
526 return return_bb; | 765 return return_bb; |
527 | 766 |
528 e = single_pred_edge (EXIT_BLOCK_PTR); | 767 e = single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun)); |
529 for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi)) | 768 for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi)) |
530 { | 769 { |
531 gimple stmt = gsi_stmt (bsi); | 770 gimple *stmt = gsi_stmt (bsi); |
532 if (gimple_code (stmt) == GIMPLE_LABEL || is_gimple_debug (stmt)) | 771 if (gimple_code (stmt) == GIMPLE_LABEL |
772 || is_gimple_debug (stmt) | |
773 || gimple_clobber_p (stmt)) | |
533 ; | 774 ; |
534 else if (gimple_code (stmt) == GIMPLE_ASSIGN | 775 else if (gimple_code (stmt) == GIMPLE_ASSIGN |
535 && found_return | 776 && found_return |
536 && gimple_assign_single_p (stmt) | 777 && gimple_assign_single_p (stmt) |
537 && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt), | 778 && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt), |
538 current_function_decl) | 779 current_function_decl) |
539 || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))) | 780 || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))) |
540 && retval == gimple_assign_lhs (stmt)) | 781 && retval == gimple_assign_lhs (stmt)) |
541 ; | 782 ; |
542 else if (gimple_code (stmt) == GIMPLE_RETURN) | 783 else if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) |
543 { | 784 { |
544 found_return = true; | 785 found_return = true; |
545 retval = gimple_return_retval (stmt); | 786 retval = gimple_return_retval (return_stmt); |
546 } | 787 } |
788 /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return | |
789 bb. */ | |
790 else if ((flag_sanitize & SANITIZE_THREAD) | |
791 && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT)) | |
792 ; | |
547 else | 793 else |
548 break; | 794 break; |
549 } | 795 } |
550 if (gsi_end_p (bsi) && found_return) | 796 if (gsi_end_p (bsi) && found_return) |
551 return_bb = e->src; | 797 return_bb = e->src; |
558 static tree | 804 static tree |
559 find_retval (basic_block return_bb) | 805 find_retval (basic_block return_bb) |
560 { | 806 { |
561 gimple_stmt_iterator bsi; | 807 gimple_stmt_iterator bsi; |
562 for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 808 for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi)) |
563 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN) | 809 if (greturn *return_stmt = dyn_cast <greturn *> (gsi_stmt (bsi))) |
564 return gimple_return_retval (gsi_stmt (bsi)); | 810 return gimple_return_retval (return_stmt); |
565 else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN) | 811 else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN |
812 && !gimple_clobber_p (gsi_stmt (bsi))) | |
566 return gimple_assign_rhs1 (gsi_stmt (bsi)); | 813 return gimple_assign_rhs1 (gsi_stmt (bsi)); |
567 return NULL; | 814 return NULL; |
568 } | 815 } |
569 | 816 |
817 /* Given return basic block RETURN_BB, see where return bounds are really | |
818 stored. */ | |
819 static tree | |
820 find_retbnd (basic_block return_bb) | |
821 { | |
822 gimple_stmt_iterator bsi; | |
823 for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi); gsi_prev (&bsi)) | |
824 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN) | |
825 return gimple_return_retbnd (gsi_stmt (bsi)); | |
826 return NULL; | |
827 } | |
828 | |
570 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic | 829 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic |
571 variable, mark it as used in bitmap passed via DATA. | 830 variable, mark it as used in bitmap passed via DATA. |
572 Return true when access to T prevents splitting the function. */ | 831 Return true when access to T prevents splitting the function. */ |
573 | 832 |
574 static bool | 833 static bool |
575 mark_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data) | 834 mark_nonssa_use (gimple *, tree t, tree, void *data) |
576 { | 835 { |
577 t = get_base_address (t); | 836 t = get_base_address (t); |
578 | 837 |
579 if (!t || is_gimple_reg (t)) | 838 if (!t || is_gimple_reg (t)) |
580 return false; | 839 return false; |
587 fprintf (dump_file, | 846 fprintf (dump_file, |
588 "Cannot split: use of non-ssa function parameter.\n"); | 847 "Cannot split: use of non-ssa function parameter.\n"); |
589 return true; | 848 return true; |
590 } | 849 } |
591 | 850 |
592 if ((TREE_CODE (t) == VAR_DECL | 851 if ((VAR_P (t) && auto_var_in_fn_p (t, current_function_decl)) |
593 && auto_var_in_fn_p (t, current_function_decl)) | |
594 || TREE_CODE (t) == RESULT_DECL | 852 || TREE_CODE (t) == RESULT_DECL |
595 || TREE_CODE (t) == LABEL_DECL) | 853 || (TREE_CODE (t) == LABEL_DECL && FORCED_LABEL (t))) |
596 bitmap_set_bit ((bitmap)data, DECL_UID (t)); | 854 bitmap_set_bit ((bitmap)data, DECL_UID (t)); |
597 | 855 |
598 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want | 856 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want |
599 to pretend that the value pointed to is actual result decl. */ | 857 to pretend that the value pointed to is actual result decl. */ |
600 if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t)) | 858 if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t)) |
601 && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME | 859 && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME |
860 && SSA_NAME_VAR (TREE_OPERAND (t, 0)) | |
602 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL | 861 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL |
603 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 862 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
604 return | 863 return |
605 bitmap_bit_p ((bitmap)data, | 864 bitmap_bit_p ((bitmap)data, |
606 DECL_UID (DECL_RESULT (current_function_decl))); | 865 DECL_UID (DECL_RESULT (current_function_decl))); |
621 static bool | 880 static bool |
622 visit_bb (basic_block bb, basic_block return_bb, | 881 visit_bb (basic_block bb, basic_block return_bb, |
623 bitmap set_ssa_names, bitmap used_ssa_names, | 882 bitmap set_ssa_names, bitmap used_ssa_names, |
624 bitmap non_ssa_vars) | 883 bitmap non_ssa_vars) |
625 { | 884 { |
626 gimple_stmt_iterator bsi; | |
627 edge e; | 885 edge e; |
628 edge_iterator ei; | 886 edge_iterator ei; |
629 bool can_split = true; | 887 bool can_split = true; |
630 | 888 |
631 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 889 for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi); |
632 { | 890 gsi_next (&bsi)) |
633 gimple stmt = gsi_stmt (bsi); | 891 { |
892 gimple *stmt = gsi_stmt (bsi); | |
634 tree op; | 893 tree op; |
635 ssa_op_iter iter; | 894 ssa_op_iter iter; |
636 tree decl; | 895 tree decl; |
637 | 896 |
638 if (is_gimple_debug (stmt)) | 897 if (is_gimple_debug (stmt)) |
898 continue; | |
899 | |
900 if (gimple_clobber_p (stmt)) | |
639 continue; | 901 continue; |
640 | 902 |
641 /* FIXME: We can split regions containing EH. We can not however | 903 /* FIXME: We can split regions containing EH. We can not however |
642 split RESX, EH_DISPATCH and EH_POINTER referring to same region | 904 split RESX, EH_DISPATCH and EH_POINTER referring to same region |
643 into different partitions. This would require tracking of | 905 into different partitions. This would require tracking of |
692 can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars, | 954 can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars, |
693 mark_nonssa_use, | 955 mark_nonssa_use, |
694 mark_nonssa_use, | 956 mark_nonssa_use, |
695 mark_nonssa_use); | 957 mark_nonssa_use); |
696 } | 958 } |
697 for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 959 for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi); |
698 { | 960 gsi_next (&bsi)) |
699 gimple stmt = gsi_stmt (bsi); | 961 { |
962 gphi *stmt = bsi.phi (); | |
700 unsigned int i; | 963 unsigned int i; |
701 | 964 |
702 if (is_gimple_debug (stmt)) | 965 if (virtual_operand_p (gimple_phi_result (stmt))) |
703 continue; | |
704 if (!is_gimple_reg (gimple_phi_result (stmt))) | |
705 continue; | 966 continue; |
706 bitmap_set_bit (set_ssa_names, | 967 bitmap_set_bit (set_ssa_names, |
707 SSA_NAME_VERSION (gimple_phi_result (stmt))); | 968 SSA_NAME_VERSION (gimple_phi_result (stmt))); |
708 for (i = 0; i < gimple_phi_num_args (stmt); i++) | 969 for (i = 0; i < gimple_phi_num_args (stmt); i++) |
709 { | 970 { |
718 } | 979 } |
719 /* Record also uses coming from PHI operand in return BB. */ | 980 /* Record also uses coming from PHI operand in return BB. */ |
720 FOR_EACH_EDGE (e, ei, bb->succs) | 981 FOR_EACH_EDGE (e, ei, bb->succs) |
721 if (e->dest == return_bb) | 982 if (e->dest == return_bb) |
722 { | 983 { |
723 for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 984 for (gphi_iterator bsi = gsi_start_phis (return_bb); |
985 !gsi_end_p (bsi); | |
986 gsi_next (&bsi)) | |
724 { | 987 { |
725 gimple stmt = gsi_stmt (bsi); | 988 gphi *stmt = bsi.phi (); |
726 tree op = gimple_phi_arg_def (stmt, e->dest_idx); | 989 tree op = gimple_phi_arg_def (stmt, e->dest_idx); |
727 | 990 |
728 if (is_gimple_debug (stmt)) | 991 if (virtual_operand_p (gimple_phi_result (stmt))) |
729 continue; | |
730 if (!is_gimple_reg (gimple_phi_result (stmt))) | |
731 continue; | 992 continue; |
732 if (TREE_CODE (op) == SSA_NAME) | 993 if (TREE_CODE (op) == SSA_NAME) |
733 bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op)); | 994 bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op)); |
734 else | 995 else |
735 can_split &= !mark_nonssa_use (stmt, op, non_ssa_vars); | 996 can_split &= !mark_nonssa_use (stmt, op, op, non_ssa_vars); |
736 } | 997 } |
737 } | 998 } |
738 return can_split; | 999 return can_split; |
739 } | 1000 } |
740 | 1001 |
741 /* Stack entry for recursive DFS walk in find_split_point. */ | 1002 /* Stack entry for recursive DFS walk in find_split_point. */ |
742 | 1003 |
743 typedef struct | 1004 struct stack_entry |
744 { | 1005 { |
745 /* Basic block we are examining. */ | 1006 /* Basic block we are examining. */ |
746 basic_block bb; | 1007 basic_block bb; |
747 | 1008 |
748 /* SSA names set and used by the BB and all BBs reachable | 1009 /* SSA names set and used by the BB and all BBs reachable |
764 /* Overall time and size of all BBs reached from this BB in DFS walk. */ | 1025 /* Overall time and size of all BBs reached from this BB in DFS walk. */ |
765 int overall_time, overall_size; | 1026 int overall_time, overall_size; |
766 | 1027 |
767 /* When false we can not split on this BB. */ | 1028 /* When false we can not split on this BB. */ |
768 bool can_split; | 1029 bool can_split; |
769 } stack_entry; | 1030 }; |
770 DEF_VEC_O(stack_entry); | |
771 DEF_VEC_ALLOC_O(stack_entry,heap); | |
772 | 1031 |
773 | 1032 |
774 /* Find all articulations and call consider_split on them. | 1033 /* Find all articulations and call consider_split on them. |
775 OVERALL_TIME and OVERALL_SIZE is time and size of the function. | 1034 OVERALL_TIME and OVERALL_SIZE is time and size of the function. |
776 | 1035 |
787 The algorithm finds articulation after visiting the whole component | 1046 The algorithm finds articulation after visiting the whole component |
788 reachable by it. This makes it convenient to collect information about | 1047 reachable by it. This makes it convenient to collect information about |
789 the component used by consider_split. */ | 1048 the component used by consider_split. */ |
790 | 1049 |
791 static void | 1050 static void |
792 find_split_points (int overall_time, int overall_size) | 1051 find_split_points (basic_block return_bb, int overall_time, int overall_size) |
793 { | 1052 { |
794 stack_entry first; | 1053 stack_entry first; |
795 VEC(stack_entry, heap) *stack = NULL; | 1054 vec<stack_entry> stack = vNULL; |
796 basic_block bb; | 1055 basic_block bb; |
797 basic_block return_bb = find_return_bb (); | |
798 struct split_point current; | 1056 struct split_point current; |
799 | 1057 |
800 current.header_time = overall_time; | 1058 current.header_time = overall_time; |
801 current.header_size = overall_size; | 1059 current.header_size = overall_size; |
802 current.split_time = 0; | 1060 current.split_time = 0; |
803 current.split_size = 0; | 1061 current.split_size = 0; |
804 current.ssa_names_to_pass = BITMAP_ALLOC (NULL); | 1062 current.ssa_names_to_pass = BITMAP_ALLOC (NULL); |
805 | 1063 |
806 first.bb = ENTRY_BLOCK_PTR; | 1064 first.bb = ENTRY_BLOCK_PTR_FOR_FN (cfun); |
807 first.edge_num = 0; | 1065 first.edge_num = 0; |
808 first.overall_time = 0; | 1066 first.overall_time = 0; |
809 first.overall_size = 0; | 1067 first.overall_size = 0; |
810 first.earliest = INT_MAX; | 1068 first.earliest = INT_MAX; |
811 first.set_ssa_names = 0; | 1069 first.set_ssa_names = 0; |
812 first.used_ssa_names = 0; | 1070 first.used_ssa_names = 0; |
1071 first.non_ssa_vars = 0; | |
813 first.bbs_visited = 0; | 1072 first.bbs_visited = 0; |
814 VEC_safe_push (stack_entry, heap, stack, &first); | 1073 first.can_split = false; |
815 ENTRY_BLOCK_PTR->aux = (void *)(intptr_t)-1; | 1074 stack.safe_push (first); |
816 | 1075 ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(intptr_t)-1; |
817 while (!VEC_empty (stack_entry, stack)) | 1076 |
818 { | 1077 while (!stack.is_empty ()) |
819 stack_entry *entry = VEC_last (stack_entry, stack); | 1078 { |
1079 stack_entry *entry = &stack.last (); | |
820 | 1080 |
821 /* We are walking an acyclic graph, so edge_num counts | 1081 /* We are walking an acyclic graph, so edge_num counts |
822 succ and pred edges together. However when considering | 1082 succ and pred edges together. However when considering |
823 articulation, we want to have processed everything reachable | 1083 articulation, we want to have processed everything reachable |
824 from articulation but nothing that reaches into it. */ | 1084 from articulation but nothing that reaches into it. */ |
825 if (entry->edge_num == EDGE_COUNT (entry->bb->succs) | 1085 if (entry->edge_num == EDGE_COUNT (entry->bb->succs) |
826 && entry->bb != ENTRY_BLOCK_PTR) | 1086 && entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun)) |
827 { | 1087 { |
828 int pos = VEC_length (stack_entry, stack); | 1088 int pos = stack.length (); |
829 entry->can_split &= visit_bb (entry->bb, return_bb, | 1089 entry->can_split &= visit_bb (entry->bb, return_bb, |
830 entry->set_ssa_names, | 1090 entry->set_ssa_names, |
831 entry->used_ssa_names, | 1091 entry->used_ssa_names, |
832 entry->non_ssa_vars); | 1092 entry->non_ssa_vars); |
833 if (pos <= entry->earliest && !entry->can_split | 1093 if (pos <= entry->earliest && !entry->can_split |
873 } | 1133 } |
874 | 1134 |
875 entry->edge_num++; | 1135 entry->edge_num++; |
876 | 1136 |
877 /* New BB to visit, push it to the stack. */ | 1137 /* New BB to visit, push it to the stack. */ |
878 if (dest != return_bb && dest != EXIT_BLOCK_PTR | 1138 if (dest != return_bb && dest != EXIT_BLOCK_PTR_FOR_FN (cfun) |
879 && !dest->aux) | 1139 && !dest->aux) |
880 { | 1140 { |
881 stack_entry new_entry; | 1141 stack_entry new_entry; |
882 | 1142 |
883 new_entry.bb = dest; | 1143 new_entry.bb = dest; |
884 new_entry.edge_num = 0; | 1144 new_entry.edge_num = 0; |
885 new_entry.overall_time | 1145 new_entry.overall_time |
886 = VEC_index (bb_info, bb_info_vec, dest->index)->time; | 1146 = bb_info_vec[dest->index].time; |
887 new_entry.overall_size | 1147 new_entry.overall_size |
888 = VEC_index (bb_info, bb_info_vec, dest->index)->size; | 1148 = bb_info_vec[dest->index].size; |
889 new_entry.earliest = INT_MAX; | 1149 new_entry.earliest = INT_MAX; |
890 new_entry.set_ssa_names = BITMAP_ALLOC (NULL); | 1150 new_entry.set_ssa_names = BITMAP_ALLOC (NULL); |
891 new_entry.used_ssa_names = BITMAP_ALLOC (NULL); | 1151 new_entry.used_ssa_names = BITMAP_ALLOC (NULL); |
892 new_entry.bbs_visited = BITMAP_ALLOC (NULL); | 1152 new_entry.bbs_visited = BITMAP_ALLOC (NULL); |
893 new_entry.non_ssa_vars = BITMAP_ALLOC (NULL); | 1153 new_entry.non_ssa_vars = BITMAP_ALLOC (NULL); |
894 new_entry.can_split = true; | 1154 new_entry.can_split = true; |
895 bitmap_set_bit (new_entry.bbs_visited, dest->index); | 1155 bitmap_set_bit (new_entry.bbs_visited, dest->index); |
896 VEC_safe_push (stack_entry, heap, stack, &new_entry); | 1156 stack.safe_push (new_entry); |
897 dest->aux = (void *)(intptr_t)VEC_length (stack_entry, stack); | 1157 dest->aux = (void *)(intptr_t)stack.length (); |
898 } | 1158 } |
899 /* Back edge found, record the earliest point. */ | 1159 /* Back edge found, record the earliest point. */ |
900 else if ((intptr_t)dest->aux > 0 | 1160 else if ((intptr_t)dest->aux > 0 |
901 && (intptr_t)dest->aux < entry->earliest) | 1161 && (intptr_t)dest->aux < entry->earliest) |
902 entry->earliest = (intptr_t)dest->aux; | 1162 entry->earliest = (intptr_t)dest->aux; |
903 } | 1163 } |
904 /* We are done with examining the edges. Pop off the value from stack | 1164 /* We are done with examining the edges. Pop off the value from stack |
905 and merge stuff we accumulate during the walk. */ | 1165 and merge stuff we accumulate during the walk. */ |
906 else if (entry->bb != ENTRY_BLOCK_PTR) | 1166 else if (entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun)) |
907 { | 1167 { |
908 stack_entry *prev = VEC_index (stack_entry, stack, | 1168 stack_entry *prev = &stack[stack.length () - 2]; |
909 VEC_length (stack_entry, stack) - 2); | |
910 | 1169 |
911 entry->bb->aux = (void *)(intptr_t)-1; | 1170 entry->bb->aux = (void *)(intptr_t)-1; |
912 prev->can_split &= entry->can_split; | 1171 prev->can_split &= entry->can_split; |
913 if (prev->set_ssa_names) | 1172 if (prev->set_ssa_names) |
914 { | 1173 { |
923 prev->overall_size += entry->overall_size; | 1182 prev->overall_size += entry->overall_size; |
924 BITMAP_FREE (entry->set_ssa_names); | 1183 BITMAP_FREE (entry->set_ssa_names); |
925 BITMAP_FREE (entry->used_ssa_names); | 1184 BITMAP_FREE (entry->used_ssa_names); |
926 BITMAP_FREE (entry->bbs_visited); | 1185 BITMAP_FREE (entry->bbs_visited); |
927 BITMAP_FREE (entry->non_ssa_vars); | 1186 BITMAP_FREE (entry->non_ssa_vars); |
928 VEC_pop (stack_entry, stack); | 1187 stack.pop (); |
929 } | 1188 } |
930 else | 1189 else |
931 VEC_pop (stack_entry, stack); | 1190 stack.pop (); |
932 } | 1191 } |
933 ENTRY_BLOCK_PTR->aux = NULL; | 1192 ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = NULL; |
934 FOR_EACH_BB (bb) | 1193 FOR_EACH_BB_FN (bb, cfun) |
935 bb->aux = NULL; | 1194 bb->aux = NULL; |
936 VEC_free (stack_entry, heap, stack); | 1195 stack.release (); |
937 BITMAP_FREE (current.ssa_names_to_pass); | 1196 BITMAP_FREE (current.ssa_names_to_pass); |
938 } | 1197 } |
939 | 1198 |
940 /* Split function at SPLIT_POINT. */ | 1199 /* Split function at SPLIT_POINT. */ |
941 | 1200 |
942 static void | 1201 static void |
943 split_function (struct split_point *split_point) | 1202 split_function (basic_block return_bb, struct split_point *split_point, |
944 { | 1203 bool add_tsan_func_exit) |
945 VEC (tree, heap) *args_to_pass = NULL; | 1204 { |
946 bitmap args_to_skip = BITMAP_ALLOC (NULL); | 1205 vec<tree> args_to_pass = vNULL; |
1206 bitmap args_to_skip; | |
947 tree parm; | 1207 tree parm; |
948 int num = 0; | 1208 int num = 0; |
949 struct cgraph_node *node; | 1209 cgraph_node *node, *cur_node = cgraph_node::get (current_function_decl); |
950 basic_block return_bb = find_return_bb (); | |
951 basic_block call_bb; | 1210 basic_block call_bb; |
952 gimple_stmt_iterator gsi; | 1211 gcall *call, *tsan_func_exit_call = NULL; |
953 gimple call; | |
954 edge e; | 1212 edge e; |
955 edge_iterator ei; | 1213 edge_iterator ei; |
956 tree retval = NULL, real_retval = NULL; | 1214 tree retval = NULL, real_retval = NULL, retbnd = NULL; |
957 bool split_part_return_p = false; | 1215 bool with_bounds = chkp_function_instrumented_p (current_function_decl); |
958 gimple last_stmt = NULL; | 1216 gimple *last_stmt = NULL; |
959 bool conv_needed = false; | |
960 unsigned int i; | 1217 unsigned int i; |
961 tree arg; | 1218 tree arg, ddef; |
962 | 1219 |
963 if (dump_file) | 1220 if (dump_file) |
964 { | 1221 { |
965 fprintf (dump_file, "\n\nSplitting function at:\n"); | 1222 fprintf (dump_file, "\n\nSplitting function at:\n"); |
966 dump_split_point (dump_file, split_point); | 1223 dump_split_point (dump_file, split_point); |
967 } | 1224 } |
1225 | |
1226 if (cur_node->local.can_change_signature) | |
1227 args_to_skip = BITMAP_ALLOC (NULL); | |
1228 else | |
1229 args_to_skip = NULL; | |
968 | 1230 |
969 /* Collect the parameters of new function and args_to_skip bitmap. */ | 1231 /* Collect the parameters of new function and args_to_skip bitmap. */ |
970 for (parm = DECL_ARGUMENTS (current_function_decl); | 1232 for (parm = DECL_ARGUMENTS (current_function_decl); |
971 parm; parm = DECL_CHAIN (parm), num++) | 1233 parm; parm = DECL_CHAIN (parm), num++) |
972 if (!is_gimple_reg (parm) | 1234 if (args_to_skip |
973 || !gimple_default_def (cfun, parm) | 1235 && (!is_gimple_reg (parm) |
974 || !bitmap_bit_p (split_point->ssa_names_to_pass, | 1236 || (ddef = ssa_default_def (cfun, parm)) == NULL_TREE |
975 SSA_NAME_VERSION (gimple_default_def (cfun, parm)))) | 1237 || !bitmap_bit_p (split_point->ssa_names_to_pass, |
1238 SSA_NAME_VERSION (ddef)))) | |
976 bitmap_set_bit (args_to_skip, num); | 1239 bitmap_set_bit (args_to_skip, num); |
977 else | 1240 else |
978 { | 1241 { |
979 arg = gimple_default_def (cfun, parm); | 1242 /* This parm might not have been used up to now, but is going to be |
980 if (TYPE_MAIN_VARIANT (DECL_ARG_TYPE (parm)) | 1243 used, hence register it. */ |
981 != TYPE_MAIN_VARIANT (TREE_TYPE (arg))) | 1244 if (is_gimple_reg (parm)) |
982 { | 1245 arg = get_or_create_ssa_default_def (cfun, parm); |
983 conv_needed = true; | 1246 else |
984 arg = fold_convert (DECL_ARG_TYPE (parm), arg); | 1247 arg = parm; |
985 } | 1248 |
986 VEC_safe_push (tree, heap, args_to_pass, arg); | 1249 if (!useless_type_conversion_p (DECL_ARG_TYPE (parm), TREE_TYPE (arg))) |
1250 arg = fold_convert (DECL_ARG_TYPE (parm), arg); | |
1251 args_to_pass.safe_push (arg); | |
987 } | 1252 } |
988 | 1253 |
989 /* See if the split function will return. */ | 1254 /* See if the split function will return. */ |
1255 bool split_part_return_p = false; | |
990 FOR_EACH_EDGE (e, ei, return_bb->preds) | 1256 FOR_EACH_EDGE (e, ei, return_bb->preds) |
991 if (bitmap_bit_p (split_point->split_bbs, e->src->index)) | 1257 { |
992 break; | 1258 if (bitmap_bit_p (split_point->split_bbs, e->src->index)) |
993 if (e) | 1259 split_part_return_p = true; |
994 split_part_return_p = true; | 1260 } |
995 | 1261 |
996 /* Add return block to what will become the split function. | 1262 /* Add return block to what will become the split function. |
997 We do not return; no return block is needed. */ | 1263 We do not return; no return block is needed. */ |
998 if (!split_part_return_p) | 1264 if (!split_part_return_p) |
999 ; | 1265 ; |
1000 /* We have no return block, so nothing is needed. */ | 1266 /* We have no return block, so nothing is needed. */ |
1001 else if (return_bb == EXIT_BLOCK_PTR) | 1267 else if (return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)) |
1002 ; | 1268 ; |
1003 /* When we do not want to return value, we need to construct | 1269 /* When we do not want to return value, we need to construct |
1004 new return block with empty return statement. | 1270 new return block with empty return statement. |
1005 FIXME: Once we are able to change return type, we should change function | 1271 FIXME: Once we are able to change return type, we should change function |
1006 to return void instead of just outputting function with undefined return | 1272 to return void instead of just outputting function with undefined return |
1007 value. For structures this affects quality of codegen. */ | 1273 value. For structures this affects quality of codegen. */ |
1008 else if (!split_point->split_part_set_retval | 1274 else if ((retval = find_retval (return_bb)) |
1009 && find_retval (return_bb)) | 1275 && !split_point->split_part_set_retval) |
1010 { | 1276 { |
1011 bool redirected = true; | 1277 bool redirected = true; |
1012 basic_block new_return_bb = create_basic_block (NULL, 0, return_bb); | 1278 basic_block new_return_bb = create_basic_block (NULL, 0, return_bb); |
1013 gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb); | 1279 gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb); |
1014 gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT); | 1280 gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT); |
1281 new_return_bb->count = profile_count::zero (); | |
1015 while (redirected) | 1282 while (redirected) |
1016 { | 1283 { |
1017 redirected = false; | 1284 redirected = false; |
1018 FOR_EACH_EDGE (e, ei, return_bb->preds) | 1285 FOR_EACH_EDGE (e, ei, return_bb->preds) |
1019 if (bitmap_bit_p (split_point->split_bbs, e->src->index)) | 1286 if (bitmap_bit_p (split_point->split_bbs, e->src->index)) |
1020 { | 1287 { |
1021 new_return_bb->count += e->count; | |
1022 new_return_bb->frequency += EDGE_FREQUENCY (e); | 1288 new_return_bb->frequency += EDGE_FREQUENCY (e); |
1023 redirect_edge_and_branch (e, new_return_bb); | 1289 redirect_edge_and_branch (e, new_return_bb); |
1024 redirected = true; | 1290 redirected = true; |
1025 break; | 1291 break; |
1026 } | 1292 } |
1027 } | 1293 } |
1028 e = make_edge (new_return_bb, EXIT_BLOCK_PTR, 0); | 1294 e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); |
1029 e->probability = REG_BR_PROB_BASE; | 1295 add_bb_to_loop (new_return_bb, current_loops->tree_root); |
1030 e->count = new_return_bb->count; | |
1031 bitmap_set_bit (split_point->split_bbs, new_return_bb->index); | 1296 bitmap_set_bit (split_point->split_bbs, new_return_bb->index); |
1297 retbnd = find_retbnd (return_bb); | |
1032 } | 1298 } |
1033 /* When we pass around the value, use existing return block. */ | 1299 /* When we pass around the value, use existing return block. */ |
1034 else | 1300 else |
1035 bitmap_set_bit (split_point->split_bbs, return_bb->index); | 1301 { |
1302 bitmap_set_bit (split_point->split_bbs, return_bb->index); | |
1303 retbnd = find_retbnd (return_bb); | |
1304 } | |
1036 | 1305 |
1037 /* If RETURN_BB has virtual operand PHIs, they must be removed and the | 1306 /* If RETURN_BB has virtual operand PHIs, they must be removed and the |
1038 virtual operand marked for renaming as we change the CFG in a way that | 1307 virtual operand marked for renaming as we change the CFG in a way that |
1039 tree-inline is not able to compensate for. | 1308 tree-inline is not able to compensate for. |
1040 | 1309 |
1041 Note this can happen whether or not we have a return value. If we have | 1310 Note this can happen whether or not we have a return value. If we have |
1042 a return value, then RETURN_BB may have PHIs for real operands too. */ | 1311 a return value, then RETURN_BB may have PHIs for real operands too. */ |
1043 if (return_bb != EXIT_BLOCK_PTR) | 1312 if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
1044 { | 1313 { |
1045 for (gsi = gsi_start_phis (return_bb); !gsi_end_p (gsi);) | 1314 bool phi_p = false; |
1046 { | 1315 for (gphi_iterator gsi = gsi_start_phis (return_bb); |
1047 gimple stmt = gsi_stmt (gsi); | 1316 !gsi_end_p (gsi);) |
1048 if (is_gimple_reg (gimple_phi_result (stmt))) | 1317 { |
1318 gphi *stmt = gsi.phi (); | |
1319 if (!virtual_operand_p (gimple_phi_result (stmt))) | |
1049 { | 1320 { |
1050 gsi_next (&gsi); | 1321 gsi_next (&gsi); |
1051 continue; | 1322 continue; |
1052 } | 1323 } |
1053 mark_virtual_phi_result_for_renaming (stmt); | 1324 mark_virtual_phi_result_for_renaming (stmt); |
1054 remove_phi_node (&gsi, true); | 1325 remove_phi_node (&gsi, true); |
1055 } | 1326 phi_p = true; |
1327 } | |
1328 /* In reality we have to rename the reaching definition of the | |
1329 virtual operand at return_bb as we will eventually release it | |
1330 when we remove the code region we outlined. | |
1331 So we have to rename all immediate virtual uses of that region | |
1332 if we didn't see a PHI definition yet. */ | |
1333 /* ??? In real reality we want to set the reaching vdef of the | |
1334 entry of the SESE region as the vuse of the call and the reaching | |
1335 vdef of the exit of the SESE region as the vdef of the call. */ | |
1336 if (!phi_p) | |
1337 for (gimple_stmt_iterator gsi = gsi_start_bb (return_bb); | |
1338 !gsi_end_p (gsi); | |
1339 gsi_next (&gsi)) | |
1340 { | |
1341 gimple *stmt = gsi_stmt (gsi); | |
1342 if (gimple_vuse (stmt)) | |
1343 { | |
1344 gimple_set_vuse (stmt, NULL_TREE); | |
1345 update_stmt (stmt); | |
1346 } | |
1347 if (gimple_vdef (stmt)) | |
1348 break; | |
1349 } | |
1056 } | 1350 } |
1057 | 1351 |
1058 /* Now create the actual clone. */ | 1352 /* Now create the actual clone. */ |
1059 rebuild_cgraph_edges (); | 1353 cgraph_edge::rebuild_edges (); |
1060 node = cgraph_function_versioning (cgraph_node (current_function_decl), | 1354 node = cur_node->create_version_clone_with_body |
1061 NULL, NULL, | 1355 (vNULL, NULL, args_to_skip, |
1062 args_to_skip, | 1356 !split_part_return_p || !split_point->split_part_set_retval, |
1063 split_point->split_bbs, | 1357 split_point->split_bbs, split_point->entry_bb, "part"); |
1064 split_point->entry_bb, "part"); | 1358 |
1359 node->split_part = true; | |
1360 | |
1361 if (cur_node->same_comdat_group) | |
1362 { | |
1363 /* TODO: call is versionable if we make sure that all | |
1364 callers are inside of a comdat group. */ | |
1365 cur_node->calls_comdat_local = 1; | |
1366 node->add_to_same_comdat_group (cur_node); | |
1367 } | |
1368 | |
1369 | |
1370 /* Let's take a time profile for splitted function. */ | |
1371 node->tp_first_run = cur_node->tp_first_run + 1; | |
1372 | |
1065 /* For usual cloning it is enough to clear builtin only when signature | 1373 /* For usual cloning it is enough to clear builtin only when signature |
1066 changes. For partial inlining we however can not expect the part | 1374 changes. For partial inlining we however can not expect the part |
1067 of builtin implementation to have same semantic as the whole. */ | 1375 of builtin implementation to have same semantic as the whole. */ |
1068 if (DECL_BUILT_IN (node->decl)) | 1376 if (DECL_BUILT_IN (node->decl)) |
1069 { | 1377 { |
1070 DECL_BUILT_IN_CLASS (node->decl) = NOT_BUILT_IN; | 1378 DECL_BUILT_IN_CLASS (node->decl) = NOT_BUILT_IN; |
1071 DECL_FUNCTION_CODE (node->decl) = (enum built_in_function) 0; | 1379 DECL_FUNCTION_CODE (node->decl) = (enum built_in_function) 0; |
1072 } | 1380 } |
1073 cgraph_node_remove_callees (cgraph_node (current_function_decl)); | 1381 |
1382 /* If return_bb contains any clobbers that refer to SSA_NAMEs | |
1383 set in the split part, remove them. Also reset debug stmts that | |
1384 refer to SSA_NAMEs set in the split part. */ | |
1385 if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) | |
1386 { | |
1387 gimple_stmt_iterator gsi = gsi_start_bb (return_bb); | |
1388 while (!gsi_end_p (gsi)) | |
1389 { | |
1390 tree op; | |
1391 ssa_op_iter iter; | |
1392 gimple *stmt = gsi_stmt (gsi); | |
1393 bool remove = false; | |
1394 if (gimple_clobber_p (stmt) || is_gimple_debug (stmt)) | |
1395 FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE) | |
1396 { | |
1397 basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op)); | |
1398 if (op != retval | |
1399 && bb | |
1400 && bb != return_bb | |
1401 && bitmap_bit_p (split_point->split_bbs, bb->index)) | |
1402 { | |
1403 if (is_gimple_debug (stmt)) | |
1404 { | |
1405 gimple_debug_bind_reset_value (stmt); | |
1406 update_stmt (stmt); | |
1407 } | |
1408 else | |
1409 remove = true; | |
1410 break; | |
1411 } | |
1412 } | |
1413 if (remove) | |
1414 gsi_remove (&gsi, true); | |
1415 else | |
1416 gsi_next (&gsi); | |
1417 } | |
1418 } | |
1419 | |
1420 /* If the original function is instrumented then it's | |
1421 part is also instrumented. */ | |
1422 if (with_bounds) | |
1423 chkp_function_mark_instrumented (node->decl); | |
1424 | |
1425 /* If the original function is declared inline, there is no point in issuing | |
1426 a warning for the non-inlinable part. */ | |
1427 DECL_NO_INLINE_WARNING_P (node->decl) = 1; | |
1428 cur_node->remove_callees (); | |
1429 cur_node->remove_all_references (); | |
1074 if (!split_part_return_p) | 1430 if (!split_part_return_p) |
1075 TREE_THIS_VOLATILE (node->decl) = 1; | 1431 TREE_THIS_VOLATILE (node->decl) = 1; |
1076 if (dump_file) | 1432 if (dump_file) |
1077 dump_function_to_file (node->decl, dump_file, dump_flags); | 1433 dump_function_to_file (node->decl, dump_file, dump_flags); |
1078 | 1434 |
1079 /* Create the basic block we place call into. It is the entry basic block | 1435 /* Create the basic block we place call into. It is the entry basic block |
1080 split after last label. */ | 1436 split after last label. */ |
1081 call_bb = split_point->entry_bb; | 1437 call_bb = split_point->entry_bb; |
1082 for (gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);) | 1438 for (gimple_stmt_iterator gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);) |
1083 if (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL) | 1439 if (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL) |
1084 { | 1440 { |
1085 last_stmt = gsi_stmt (gsi); | 1441 last_stmt = gsi_stmt (gsi); |
1086 gsi_next (&gsi); | 1442 gsi_next (&gsi); |
1087 } | 1443 } |
1089 break; | 1445 break; |
1090 e = split_block (split_point->entry_bb, last_stmt); | 1446 e = split_block (split_point->entry_bb, last_stmt); |
1091 remove_edge (e); | 1447 remove_edge (e); |
1092 | 1448 |
1093 /* Produce the call statement. */ | 1449 /* Produce the call statement. */ |
1094 gsi = gsi_last_bb (call_bb); | 1450 gimple_stmt_iterator gsi = gsi_last_bb (call_bb); |
1095 if (conv_needed) | 1451 FOR_EACH_VEC_ELT (args_to_pass, i, arg) |
1096 FOR_EACH_VEC_ELT (tree, args_to_pass, i, arg) | 1452 if (!is_gimple_val (arg)) |
1097 if (!is_gimple_val (arg)) | 1453 { |
1098 { | 1454 arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE, |
1099 arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE, | 1455 false, GSI_CONTINUE_LINKING); |
1100 false, GSI_NEW_STMT); | 1456 args_to_pass[i] = arg; |
1101 VEC_replace (tree, args_to_pass, i, arg); | 1457 } |
1102 } | |
1103 call = gimple_build_call_vec (node->decl, args_to_pass); | 1458 call = gimple_build_call_vec (node->decl, args_to_pass); |
1459 gimple_call_set_with_bounds (call, with_bounds); | |
1104 gimple_set_block (call, DECL_INITIAL (current_function_decl)); | 1460 gimple_set_block (call, DECL_INITIAL (current_function_decl)); |
1461 args_to_pass.release (); | |
1462 | |
1463 /* For optimized away parameters, add on the caller side | |
1464 before the call | |
1465 DEBUG D#X => parm_Y(D) | |
1466 stmts and associate D#X with parm in decl_debug_args_lookup | |
1467 vector to say for debug info that if parameter parm had been passed, | |
1468 it would have value parm_Y(D). */ | |
1469 if (args_to_skip) | |
1470 { | |
1471 vec<tree, va_gc> **debug_args = NULL; | |
1472 unsigned i = 0, len = 0; | |
1473 if (MAY_HAVE_DEBUG_STMTS) | |
1474 { | |
1475 debug_args = decl_debug_args_lookup (node->decl); | |
1476 if (debug_args) | |
1477 len = vec_safe_length (*debug_args); | |
1478 } | |
1479 for (parm = DECL_ARGUMENTS (current_function_decl), num = 0; | |
1480 parm; parm = DECL_CHAIN (parm), num++) | |
1481 if (bitmap_bit_p (args_to_skip, num) && is_gimple_reg (parm)) | |
1482 { | |
1483 tree ddecl; | |
1484 gimple *def_temp; | |
1485 | |
1486 /* This needs to be done even without MAY_HAVE_DEBUG_STMTS, | |
1487 otherwise if it didn't exist before, we'd end up with | |
1488 different SSA_NAME_VERSIONs between -g and -g0. */ | |
1489 arg = get_or_create_ssa_default_def (cfun, parm); | |
1490 if (!MAY_HAVE_DEBUG_STMTS || debug_args == NULL) | |
1491 continue; | |
1492 | |
1493 while (i < len && (**debug_args)[i] != DECL_ORIGIN (parm)) | |
1494 i += 2; | |
1495 if (i >= len) | |
1496 continue; | |
1497 ddecl = (**debug_args)[i + 1]; | |
1498 def_temp | |
1499 = gimple_build_debug_bind (ddecl, unshare_expr (arg), call); | |
1500 gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT); | |
1501 } | |
1502 } | |
1105 | 1503 |
1106 /* We avoid address being taken on any variable used by split part, | 1504 /* We avoid address being taken on any variable used by split part, |
1107 so return slot optimization is always possible. Moreover this is | 1505 so return slot optimization is always possible. Moreover this is |
1108 required to make DECL_BY_REFERENCE work. */ | 1506 required to make DECL_BY_REFERENCE work. */ |
1109 if (aggregate_value_p (DECL_RESULT (current_function_decl), | 1507 if (aggregate_value_p (DECL_RESULT (current_function_decl), |
1110 TREE_TYPE (current_function_decl))) | 1508 TREE_TYPE (current_function_decl)) |
1509 && (!is_gimple_reg_type (TREE_TYPE (DECL_RESULT (current_function_decl))) | |
1510 || DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))) | |
1111 gimple_call_set_return_slot_opt (call, true); | 1511 gimple_call_set_return_slot_opt (call, true); |
1512 | |
1513 if (add_tsan_func_exit) | |
1514 tsan_func_exit_call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0); | |
1112 | 1515 |
1113 /* Update return value. This is bit tricky. When we do not return, | 1516 /* Update return value. This is bit tricky. When we do not return, |
1114 do nothing. When we return we might need to update return_bb | 1517 do nothing. When we return we might need to update return_bb |
1115 or produce a new return statement. */ | 1518 or produce a new return statement. */ |
1116 if (!split_part_return_p) | 1519 if (!split_part_return_p) |
1117 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | 1520 { |
1521 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1522 if (tsan_func_exit_call) | |
1523 gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT); | |
1524 } | |
1118 else | 1525 else |
1119 { | 1526 { |
1120 e = make_edge (call_bb, return_bb, | 1527 e = make_single_succ_edge (call_bb, return_bb, |
1121 return_bb == EXIT_BLOCK_PTR ? 0 : EDGE_FALLTHRU); | 1528 return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun) |
1122 e->count = call_bb->count; | 1529 ? 0 : EDGE_FALLTHRU); |
1123 e->probability = REG_BR_PROB_BASE; | |
1124 | 1530 |
1125 /* If there is return basic block, see what value we need to store | 1531 /* If there is return basic block, see what value we need to store |
1126 return value into and put call just before it. */ | 1532 return value into and put call just before it. */ |
1127 if (return_bb != EXIT_BLOCK_PTR) | 1533 if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
1128 { | 1534 { |
1129 real_retval = retval = find_retval (return_bb); | 1535 real_retval = retval; |
1130 | |
1131 if (real_retval && split_point->split_part_set_retval) | 1536 if (real_retval && split_point->split_part_set_retval) |
1132 { | 1537 { |
1133 gimple_stmt_iterator psi; | 1538 gphi_iterator psi; |
1134 | 1539 |
1135 /* See if we need new SSA_NAME for the result. | 1540 /* See if we need new SSA_NAME for the result. |
1136 When DECL_BY_REFERENCE is true, retval is actually pointer to | 1541 When DECL_BY_REFERENCE is true, retval is actually pointer to |
1137 return value and it is constant in whole function. */ | 1542 return value and it is constant in whole function. */ |
1138 if (TREE_CODE (retval) == SSA_NAME | 1543 if (TREE_CODE (retval) == SSA_NAME |
1139 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 1544 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
1140 { | 1545 { |
1141 retval = make_ssa_name (SSA_NAME_VAR (retval), call); | 1546 retval = copy_ssa_name (retval, call); |
1142 | 1547 |
1143 /* See if there is PHI defining return value. */ | 1548 /* See if there is PHI defining return value. */ |
1144 for (psi = gsi_start_phis (return_bb); | 1549 for (psi = gsi_start_phis (return_bb); |
1145 !gsi_end_p (psi); gsi_next (&psi)) | 1550 !gsi_end_p (psi); gsi_next (&psi)) |
1146 if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi)))) | 1551 if (!virtual_operand_p (gimple_phi_result (psi.phi ()))) |
1147 break; | 1552 break; |
1148 | 1553 |
1149 /* When there is PHI, just update its value. */ | 1554 /* When there is PHI, just update its value. */ |
1150 if (TREE_CODE (retval) == SSA_NAME | 1555 if (TREE_CODE (retval) == SSA_NAME |
1151 && !gsi_end_p (psi)) | 1556 && !gsi_end_p (psi)) |
1152 add_phi_arg (gsi_stmt (psi), retval, e, UNKNOWN_LOCATION); | 1557 add_phi_arg (psi.phi (), retval, e, UNKNOWN_LOCATION); |
1153 /* Otherwise update the return BB itself. | 1558 /* Otherwise update the return BB itself. |
1154 find_return_bb allows at most one assignment to return value, | 1559 find_return_bb allows at most one assignment to return value, |
1155 so update first statement. */ | 1560 so update first statement. */ |
1156 else | 1561 else |
1157 { | 1562 { |
1158 gimple_stmt_iterator bsi; | 1563 gimple_stmt_iterator bsi; |
1159 for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); | 1564 for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); |
1160 gsi_next (&bsi)) | 1565 gsi_next (&bsi)) |
1161 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN) | 1566 if (greturn *return_stmt |
1567 = dyn_cast <greturn *> (gsi_stmt (bsi))) | |
1162 { | 1568 { |
1163 gimple_return_set_retval (gsi_stmt (bsi), retval); | 1569 gimple_return_set_retval (return_stmt, retval); |
1164 break; | 1570 break; |
1165 } | 1571 } |
1166 else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN) | 1572 else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN |
1573 && !gimple_clobber_p (gsi_stmt (bsi))) | |
1167 { | 1574 { |
1168 gimple_assign_set_rhs1 (gsi_stmt (bsi), retval); | 1575 gimple_assign_set_rhs1 (gsi_stmt (bsi), retval); |
1169 break; | 1576 break; |
1170 } | 1577 } |
1171 update_stmt (gsi_stmt (bsi)); | 1578 update_stmt (gsi_stmt (bsi)); |
1579 /* Also adjust clobbers and debug stmts in return_bb. */ | |
1580 for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); | |
1581 gsi_next (&bsi)) | |
1582 { | |
1583 gimple *stmt = gsi_stmt (bsi); | |
1584 if (gimple_clobber_p (stmt) | |
1585 || is_gimple_debug (stmt)) | |
1586 { | |
1587 ssa_op_iter iter; | |
1588 use_operand_p use_p; | |
1589 bool update = false; | |
1590 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, | |
1591 SSA_OP_USE) | |
1592 if (USE_FROM_PTR (use_p) == real_retval) | |
1593 { | |
1594 SET_USE (use_p, retval); | |
1595 update = true; | |
1596 } | |
1597 if (update) | |
1598 update_stmt (stmt); | |
1599 } | |
1600 } | |
1601 } | |
1602 | |
1603 /* Replace retbnd with new one. */ | |
1604 if (retbnd) | |
1605 { | |
1606 gimple_stmt_iterator bsi; | |
1607 for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi); | |
1608 gsi_prev (&bsi)) | |
1609 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN) | |
1610 { | |
1611 retbnd = copy_ssa_name (retbnd, call); | |
1612 gimple_return_set_retbnd (gsi_stmt (bsi), retbnd); | |
1613 update_stmt (gsi_stmt (bsi)); | |
1614 break; | |
1615 } | |
1172 } | 1616 } |
1173 } | 1617 } |
1174 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 1618 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
1175 gimple_call_set_lhs (call, build_simple_mem_ref (retval)); | 1619 { |
1620 gimple_call_set_lhs (call, build_simple_mem_ref (retval)); | |
1621 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1622 } | |
1176 else | 1623 else |
1177 gimple_call_set_lhs (call, retval); | 1624 { |
1625 tree restype; | |
1626 restype = TREE_TYPE (DECL_RESULT (current_function_decl)); | |
1627 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1628 if (!useless_type_conversion_p (TREE_TYPE (retval), restype)) | |
1629 { | |
1630 gimple *cpy; | |
1631 tree tem = create_tmp_reg (restype); | |
1632 tem = make_ssa_name (tem, call); | |
1633 cpy = gimple_build_assign (retval, NOP_EXPR, tem); | |
1634 gsi_insert_after (&gsi, cpy, GSI_NEW_STMT); | |
1635 retval = tem; | |
1636 } | |
1637 /* Build bndret call to obtain returned bounds. */ | |
1638 if (retbnd) | |
1639 chkp_insert_retbnd_call (retbnd, retval, &gsi); | |
1640 gimple_call_set_lhs (call, retval); | |
1641 update_stmt (call); | |
1642 } | |
1178 } | 1643 } |
1179 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | 1644 else |
1645 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1646 if (tsan_func_exit_call) | |
1647 gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT); | |
1180 } | 1648 } |
1181 /* We don't use return block (there is either no return in function or | 1649 /* We don't use return block (there is either no return in function or |
1182 multiple of them). So create new basic block with return statement. | 1650 multiple of them). So create new basic block with return statement. |
1183 */ | 1651 */ |
1184 else | 1652 else |
1185 { | 1653 { |
1186 gimple ret; | 1654 greturn *ret; |
1187 if (split_point->split_part_set_retval | 1655 if (split_point->split_part_set_retval |
1188 && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl)))) | 1656 && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl)))) |
1189 { | 1657 { |
1190 retval = DECL_RESULT (current_function_decl); | 1658 retval = DECL_RESULT (current_function_decl); |
1659 | |
1660 if (chkp_function_instrumented_p (current_function_decl) | |
1661 && BOUNDED_P (retval)) | |
1662 retbnd = create_tmp_reg (pointer_bounds_type_node); | |
1191 | 1663 |
1192 /* We use temporary register to hold value when aggregate_value_p | 1664 /* We use temporary register to hold value when aggregate_value_p |
1193 is false. Similarly for DECL_BY_REFERENCE we must avoid extra | 1665 is false. Similarly for DECL_BY_REFERENCE we must avoid extra |
1194 copy. */ | 1666 copy. */ |
1195 if (!aggregate_value_p (retval, TREE_TYPE (current_function_decl)) | 1667 if (!aggregate_value_p (retval, TREE_TYPE (current_function_decl)) |
1196 && !DECL_BY_REFERENCE (retval)) | 1668 && !DECL_BY_REFERENCE (retval)) |
1197 retval = create_tmp_reg (TREE_TYPE (retval), NULL); | 1669 retval = create_tmp_reg (TREE_TYPE (retval)); |
1198 if (is_gimple_reg (retval)) | 1670 if (is_gimple_reg (retval)) |
1199 { | 1671 { |
1200 /* When returning by reference, there is only one SSA name | 1672 /* When returning by reference, there is only one SSA name |
1201 assigned to RESULT_DECL (that is pointer to return value). | 1673 assigned to RESULT_DECL (that is pointer to return value). |
1202 Look it up or create new one if it is missing. */ | 1674 Look it up or create new one if it is missing. */ |
1203 if (DECL_BY_REFERENCE (retval)) | 1675 if (DECL_BY_REFERENCE (retval)) |
1204 { | 1676 retval = get_or_create_ssa_default_def (cfun, retval); |
1205 tree retval_name; | |
1206 if ((retval_name = gimple_default_def (cfun, retval)) | |
1207 != NULL) | |
1208 retval = retval_name; | |
1209 else | |
1210 { | |
1211 retval_name = make_ssa_name (retval, | |
1212 gimple_build_nop ()); | |
1213 set_default_def (retval, retval_name); | |
1214 retval = retval_name; | |
1215 } | |
1216 } | |
1217 /* Otherwise produce new SSA name for return value. */ | 1677 /* Otherwise produce new SSA name for return value. */ |
1218 else | 1678 else |
1219 retval = make_ssa_name (retval, call); | 1679 retval = make_ssa_name (retval, call); |
1220 } | 1680 } |
1221 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) | 1681 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))) |
1222 gimple_call_set_lhs (call, build_simple_mem_ref (retval)); | 1682 gimple_call_set_lhs (call, build_simple_mem_ref (retval)); |
1223 else | 1683 else |
1224 gimple_call_set_lhs (call, retval); | 1684 gimple_call_set_lhs (call, retval); |
1685 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1225 } | 1686 } |
1226 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | 1687 else |
1688 { | |
1689 gsi_insert_after (&gsi, call, GSI_NEW_STMT); | |
1690 if (retval | |
1691 && is_gimple_reg_type (TREE_TYPE (retval)) | |
1692 && !is_gimple_val (retval)) | |
1693 { | |
1694 gassign *g | |
1695 = gimple_build_assign (make_ssa_name (TREE_TYPE (retval)), | |
1696 retval); | |
1697 retval = gimple_assign_lhs (g); | |
1698 gsi_insert_after (&gsi, g, GSI_NEW_STMT); | |
1699 } | |
1700 } | |
1701 /* Build bndret call to obtain returned bounds. */ | |
1702 if (retbnd) | |
1703 chkp_insert_retbnd_call (retbnd, retval, &gsi); | |
1704 if (tsan_func_exit_call) | |
1705 gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT); | |
1227 ret = gimple_build_return (retval); | 1706 ret = gimple_build_return (retval); |
1228 gsi_insert_after (&gsi, ret, GSI_NEW_STMT); | 1707 gsi_insert_after (&gsi, ret, GSI_NEW_STMT); |
1229 } | 1708 } |
1230 } | 1709 } |
1231 free_dominance_info (CDI_DOMINATORS); | 1710 free_dominance_info (CDI_DOMINATORS); |
1232 free_dominance_info (CDI_POST_DOMINATORS); | 1711 free_dominance_info (CDI_POST_DOMINATORS); |
1233 compute_inline_parameters (node); | 1712 compute_fn_summary (node, true); |
1234 } | 1713 } |
1235 | 1714 |
1236 /* Execute function splitting pass. */ | 1715 /* Execute function splitting pass. */ |
1237 | 1716 |
1238 static unsigned int | 1717 static unsigned int |
1240 { | 1719 { |
1241 gimple_stmt_iterator bsi; | 1720 gimple_stmt_iterator bsi; |
1242 basic_block bb; | 1721 basic_block bb; |
1243 int overall_time = 0, overall_size = 0; | 1722 int overall_time = 0, overall_size = 0; |
1244 int todo = 0; | 1723 int todo = 0; |
1245 struct cgraph_node *node = cgraph_node (current_function_decl); | 1724 struct cgraph_node *node = cgraph_node::get (current_function_decl); |
1246 | 1725 |
1247 if (flags_from_decl_or_type (current_function_decl) & ECF_NORETURN) | 1726 if (flags_from_decl_or_type (current_function_decl) |
1727 & (ECF_NORETURN|ECF_MALLOC)) | |
1248 { | 1728 { |
1249 if (dump_file) | 1729 if (dump_file) |
1250 fprintf (dump_file, "Not splitting: noreturn function.\n"); | 1730 fprintf (dump_file, "Not splitting: noreturn/malloc function.\n"); |
1251 return 0; | 1731 return 0; |
1252 } | 1732 } |
1253 if (MAIN_NAME_P (DECL_NAME (current_function_decl))) | 1733 if (MAIN_NAME_P (DECL_NAME (current_function_decl))) |
1254 { | 1734 { |
1255 if (dump_file) | 1735 if (dump_file) |
1256 fprintf (dump_file, "Not splitting: main function.\n"); | 1736 fprintf (dump_file, "Not splitting: main function.\n"); |
1257 return 0; | 1737 return 0; |
1258 } | 1738 } |
1259 /* This can be relaxed; function might become inlinable after splitting | 1739 /* This can be relaxed; function might become inlinable after splitting |
1260 away the uninlinable part. */ | 1740 away the uninlinable part. */ |
1261 if (!node->local.inlinable) | 1741 if (ipa_fn_summaries |
1742 && !ipa_fn_summaries->get (node)->inlinable) | |
1262 { | 1743 { |
1263 if (dump_file) | 1744 if (dump_file) |
1264 fprintf (dump_file, "Not splitting: not inlinable.\n"); | 1745 fprintf (dump_file, "Not splitting: not inlinable.\n"); |
1265 return 0; | 1746 return 0; |
1266 } | 1747 } |
1267 if (node->local.disregard_inline_limits) | 1748 if (DECL_DISREGARD_INLINE_LIMITS (node->decl)) |
1268 { | 1749 { |
1269 if (dump_file) | 1750 if (dump_file) |
1270 fprintf (dump_file, "Not splitting: disregarding inline limits.\n"); | 1751 fprintf (dump_file, "Not splitting: disregarding inline limits.\n"); |
1271 return 0; | 1752 return 0; |
1272 } | 1753 } |
1293 training for LTO -fprofile-use build. | 1774 training for LTO -fprofile-use build. |
1294 | 1775 |
1295 Note that we are not completely conservative about disqualifying functions | 1776 Note that we are not completely conservative about disqualifying functions |
1296 called once. It is possible that the caller is called more then once and | 1777 called once. It is possible that the caller is called more then once and |
1297 then inlining would still benefit. */ | 1778 then inlining would still benefit. */ |
1298 if ((!node->callers || !node->callers->next_caller) | 1779 if ((!node->callers |
1780 /* Local functions called once will be completely inlined most of time. */ | |
1781 || (!node->callers->next_caller && node->local.local)) | |
1299 && !node->address_taken | 1782 && !node->address_taken |
1300 && (!flag_lto || !node->local.externally_visible)) | 1783 && !node->has_aliases_p () |
1784 && (!flag_lto || !node->externally_visible)) | |
1301 { | 1785 { |
1302 if (dump_file) | 1786 if (dump_file) |
1303 fprintf (dump_file, "Not splitting: not called directly " | 1787 fprintf (dump_file, "Not splitting: not called directly " |
1304 "or called once.\n"); | 1788 "or called once.\n"); |
1305 return 0; | 1789 return 0; |
1313 fprintf (dump_file, "Not splitting: not autoinlining and function" | 1797 fprintf (dump_file, "Not splitting: not autoinlining and function" |
1314 " is not inline.\n"); | 1798 " is not inline.\n"); |
1315 return 0; | 1799 return 0; |
1316 } | 1800 } |
1317 | 1801 |
1802 /* We enforce splitting after loop headers when profile info is not | |
1803 available. */ | |
1804 if (profile_status_for_fn (cfun) != PROFILE_READ) | |
1805 mark_dfs_back_edges (); | |
1806 | |
1807 /* Initialize bitmap to track forbidden calls. */ | |
1808 forbidden_dominators = BITMAP_ALLOC (NULL); | |
1809 calculate_dominance_info (CDI_DOMINATORS); | |
1810 | |
1318 /* Compute local info about basic blocks and determine function size/time. */ | 1811 /* Compute local info about basic blocks and determine function size/time. */ |
1319 VEC_safe_grow_cleared (bb_info, heap, bb_info_vec, last_basic_block + 1); | 1812 bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1); |
1320 memset (&best_split_point, 0, sizeof (best_split_point)); | 1813 memset (&best_split_point, 0, sizeof (best_split_point)); |
1321 FOR_EACH_BB (bb) | 1814 basic_block return_bb = find_return_bb (); |
1815 int tsan_exit_found = -1; | |
1816 FOR_EACH_BB_FN (bb, cfun) | |
1322 { | 1817 { |
1323 int time = 0; | 1818 int time = 0; |
1324 int size = 0; | 1819 int size = 0; |
1325 int freq = compute_call_stmt_bb_frequency (current_function_decl, bb); | 1820 int freq = compute_call_stmt_bb_frequency (current_function_decl, bb); |
1326 | 1821 |
1328 fprintf (dump_file, "Basic block %i\n", bb->index); | 1823 fprintf (dump_file, "Basic block %i\n", bb->index); |
1329 | 1824 |
1330 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | 1825 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) |
1331 { | 1826 { |
1332 int this_time, this_size; | 1827 int this_time, this_size; |
1333 gimple stmt = gsi_stmt (bsi); | 1828 gimple *stmt = gsi_stmt (bsi); |
1334 | 1829 |
1335 this_size = estimate_num_insns (stmt, &eni_size_weights); | 1830 this_size = estimate_num_insns (stmt, &eni_size_weights); |
1336 this_time = estimate_num_insns (stmt, &eni_time_weights) * freq; | 1831 this_time = estimate_num_insns (stmt, &eni_time_weights) * freq; |
1337 size += this_size; | 1832 size += this_size; |
1338 time += this_time; | 1833 time += this_time; |
1834 check_forbidden_calls (stmt); | |
1339 | 1835 |
1340 if (dump_file && (dump_flags & TDF_DETAILS)) | 1836 if (dump_file && (dump_flags & TDF_DETAILS)) |
1341 { | 1837 { |
1342 fprintf (dump_file, " freq:%6i size:%3i time:%3i ", | 1838 fprintf (dump_file, " freq:%6i size:%3i time:%3i ", |
1343 freq, this_size, this_time); | 1839 freq, this_size, this_time); |
1344 print_gimple_stmt (dump_file, stmt, 0, 0); | 1840 print_gimple_stmt (dump_file, stmt, 0); |
1841 } | |
1842 | |
1843 if ((flag_sanitize & SANITIZE_THREAD) | |
1844 && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT)) | |
1845 { | |
1846 /* We handle TSAN_FUNC_EXIT for splitting either in the | |
1847 return_bb, or in its immediate predecessors. */ | |
1848 if ((bb != return_bb && !find_edge (bb, return_bb)) | |
1849 || (tsan_exit_found != -1 | |
1850 && tsan_exit_found != (bb != return_bb))) | |
1851 { | |
1852 if (dump_file) | |
1853 fprintf (dump_file, "Not splitting: TSAN_FUNC_EXIT" | |
1854 " in unexpected basic block.\n"); | |
1855 BITMAP_FREE (forbidden_dominators); | |
1856 bb_info_vec.release (); | |
1857 return 0; | |
1858 } | |
1859 tsan_exit_found = bb != return_bb; | |
1345 } | 1860 } |
1346 } | 1861 } |
1347 overall_time += time; | 1862 overall_time += time; |
1348 overall_size += size; | 1863 overall_size += size; |
1349 VEC_index (bb_info, bb_info_vec, bb->index)->time = time; | 1864 bb_info_vec[bb->index].time = time; |
1350 VEC_index (bb_info, bb_info_vec, bb->index)->size = size; | 1865 bb_info_vec[bb->index].size = size; |
1351 } | 1866 } |
1352 find_split_points (overall_time, overall_size); | 1867 find_split_points (return_bb, overall_time, overall_size); |
1353 if (best_split_point.split_bbs) | 1868 if (best_split_point.split_bbs) |
1354 { | 1869 { |
1355 split_function (&best_split_point); | 1870 split_function (return_bb, &best_split_point, tsan_exit_found == 1); |
1356 BITMAP_FREE (best_split_point.ssa_names_to_pass); | 1871 BITMAP_FREE (best_split_point.ssa_names_to_pass); |
1357 BITMAP_FREE (best_split_point.split_bbs); | 1872 BITMAP_FREE (best_split_point.split_bbs); |
1358 todo = TODO_update_ssa | TODO_cleanup_cfg; | 1873 todo = TODO_update_ssa | TODO_cleanup_cfg; |
1359 } | 1874 } |
1360 VEC_free (bb_info, heap, bb_info_vec); | 1875 BITMAP_FREE (forbidden_dominators); |
1361 bb_info_vec = NULL; | 1876 bb_info_vec.release (); |
1362 return todo; | 1877 return todo; |
1363 } | 1878 } |
1364 | 1879 |
1365 /* Gate function splitting pass. When doing profile feedback, we want | 1880 namespace { |
1366 to execute the pass after profiling is read. So disable one in | 1881 |
1367 early optimization. */ | 1882 const pass_data pass_data_split_functions = |
1368 | 1883 { |
1369 static bool | 1884 GIMPLE_PASS, /* type */ |
1370 gate_split_functions (void) | 1885 "fnsplit", /* name */ |
1371 { | 1886 OPTGROUP_NONE, /* optinfo_flags */ |
1887 TV_IPA_FNSPLIT, /* tv_id */ | |
1888 PROP_cfg, /* properties_required */ | |
1889 0, /* properties_provided */ | |
1890 0, /* properties_destroyed */ | |
1891 0, /* todo_flags_start */ | |
1892 0, /* todo_flags_finish */ | |
1893 }; | |
1894 | |
1895 class pass_split_functions : public gimple_opt_pass | |
1896 { | |
1897 public: | |
1898 pass_split_functions (gcc::context *ctxt) | |
1899 : gimple_opt_pass (pass_data_split_functions, ctxt) | |
1900 {} | |
1901 | |
1902 /* opt_pass methods: */ | |
1903 virtual bool gate (function *); | |
1904 virtual unsigned int execute (function *) | |
1905 { | |
1906 return execute_split_functions (); | |
1907 } | |
1908 | |
1909 }; // class pass_split_functions | |
1910 | |
1911 bool | |
1912 pass_split_functions::gate (function *) | |
1913 { | |
1914 /* When doing profile feedback, we want to execute the pass after profiling | |
1915 is read. So disable one in early optimization. */ | |
1372 return (flag_partial_inlining | 1916 return (flag_partial_inlining |
1373 && !profile_arc_flag && !flag_branch_probabilities); | 1917 && !profile_arc_flag && !flag_branch_probabilities); |
1374 } | 1918 } |
1375 | 1919 |
1376 struct gimple_opt_pass pass_split_functions = | 1920 } // anon namespace |
1377 { | 1921 |
1378 { | 1922 gimple_opt_pass * |
1379 GIMPLE_PASS, | 1923 make_pass_split_functions (gcc::context *ctxt) |
1380 "fnsplit", /* name */ | 1924 { |
1381 gate_split_functions, /* gate */ | 1925 return new pass_split_functions (ctxt); |
1382 execute_split_functions, /* execute */ | |
1383 NULL, /* sub */ | |
1384 NULL, /* next */ | |
1385 0, /* static_pass_number */ | |
1386 TV_IPA_FNSPLIT, /* tv_id */ | |
1387 PROP_cfg, /* properties_required */ | |
1388 0, /* properties_provided */ | |
1389 0, /* properties_destroyed */ | |
1390 0, /* todo_flags_start */ | |
1391 TODO_dump_func /* todo_flags_finish */ | |
1392 } | |
1393 }; | |
1394 | |
1395 /* Gate feedback driven function splitting pass. | |
1396 We don't need to split when profiling at all, we are producing | |
1397 lousy code anyway. */ | |
1398 | |
1399 static bool | |
1400 gate_feedback_split_functions (void) | |
1401 { | |
1402 return (flag_partial_inlining | |
1403 && flag_branch_probabilities); | |
1404 } | 1926 } |
1405 | 1927 |
1406 /* Execute function splitting pass. */ | 1928 /* Execute function splitting pass. */ |
1407 | 1929 |
1408 static unsigned int | 1930 static unsigned int |
1412 if (retval) | 1934 if (retval) |
1413 retval |= TODO_rebuild_cgraph_edges; | 1935 retval |= TODO_rebuild_cgraph_edges; |
1414 return retval; | 1936 return retval; |
1415 } | 1937 } |
1416 | 1938 |
1417 struct gimple_opt_pass pass_feedback_split_functions = | 1939 namespace { |
1418 { | 1940 |
1419 { | 1941 const pass_data pass_data_feedback_split_functions = |
1420 GIMPLE_PASS, | 1942 { |
1421 "feedback_fnsplit", /* name */ | 1943 GIMPLE_PASS, /* type */ |
1422 gate_feedback_split_functions, /* gate */ | 1944 "feedback_fnsplit", /* name */ |
1423 execute_feedback_split_functions, /* execute */ | 1945 OPTGROUP_NONE, /* optinfo_flags */ |
1424 NULL, /* sub */ | 1946 TV_IPA_FNSPLIT, /* tv_id */ |
1425 NULL, /* next */ | 1947 PROP_cfg, /* properties_required */ |
1426 0, /* static_pass_number */ | 1948 0, /* properties_provided */ |
1427 TV_IPA_FNSPLIT, /* tv_id */ | 1949 0, /* properties_destroyed */ |
1428 PROP_cfg, /* properties_required */ | 1950 0, /* todo_flags_start */ |
1429 0, /* properties_provided */ | 1951 0, /* todo_flags_finish */ |
1430 0, /* properties_destroyed */ | |
1431 0, /* todo_flags_start */ | |
1432 TODO_dump_func /* todo_flags_finish */ | |
1433 } | |
1434 }; | 1952 }; |
1953 | |
1954 class pass_feedback_split_functions : public gimple_opt_pass | |
1955 { | |
1956 public: | |
1957 pass_feedback_split_functions (gcc::context *ctxt) | |
1958 : gimple_opt_pass (pass_data_feedback_split_functions, ctxt) | |
1959 {} | |
1960 | |
1961 /* opt_pass methods: */ | |
1962 virtual bool gate (function *); | |
1963 virtual unsigned int execute (function *) | |
1964 { | |
1965 return execute_feedback_split_functions (); | |
1966 } | |
1967 | |
1968 }; // class pass_feedback_split_functions | |
1969 | |
1970 bool | |
1971 pass_feedback_split_functions::gate (function *) | |
1972 { | |
1973 /* We don't need to split when profiling at all, we are producing | |
1974 lousy code anyway. */ | |
1975 return (flag_partial_inlining | |
1976 && flag_branch_probabilities); | |
1977 } | |
1978 | |
1979 } // anon namespace | |
1980 | |
1981 gimple_opt_pass * | |
1982 make_pass_feedback_split_functions (gcc::context *ctxt) | |
1983 { | |
1984 return new pass_feedback_split_functions (ctxt); | |
1985 } |