Mercurial > hg > CbC > CbC_gcc
comparison gcc/ipa-inline.c @ 55:77e2b8dfacca gcc-4.4.5
update it from 4.4.3 to 4.5.0
author | ryoma <e075725@ie.u-ryukyu.ac.jp> |
---|---|
date | Fri, 12 Feb 2010 23:39:51 +0900 |
parents | a06113de4d67 |
children | b7f97abdc517 |
comparison
equal
deleted
inserted
replaced
52:c156f1bd5cd9 | 55:77e2b8dfacca |
---|---|
136 #include "coverage.h" | 136 #include "coverage.h" |
137 #include "ggc.h" | 137 #include "ggc.h" |
138 #include "tree-flow.h" | 138 #include "tree-flow.h" |
139 #include "rtl.h" | 139 #include "rtl.h" |
140 #include "ipa-prop.h" | 140 #include "ipa-prop.h" |
141 #include "except.h" | |
142 | |
143 #define MAX_TIME 1000000000 | |
141 | 144 |
142 /* Mode incremental inliner operate on: | 145 /* Mode incremental inliner operate on: |
143 | 146 |
144 In ALWAYS_INLINE only functions marked | 147 In ALWAYS_INLINE only functions marked |
145 always_inline are inlined. This mode is used after detecting cycle during | 148 always_inline are inlined. This mode is used after detecting cycle during |
150 | 153 |
151 in ALL mode, everything is inlined. This is used during flattening. */ | 154 in ALL mode, everything is inlined. This is used during flattening. */ |
152 enum inlining_mode { | 155 enum inlining_mode { |
153 INLINE_NONE = 0, | 156 INLINE_NONE = 0, |
154 INLINE_ALWAYS_INLINE, | 157 INLINE_ALWAYS_INLINE, |
158 INLINE_SIZE_NORECURSIVE, | |
155 INLINE_SIZE, | 159 INLINE_SIZE, |
156 INLINE_ALL | 160 INLINE_ALL |
157 }; | 161 }; |
158 static bool | 162 static bool |
159 cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode, | 163 cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode, |
161 | 165 |
162 | 166 |
163 /* Statistics we collect about inlining algorithm. */ | 167 /* Statistics we collect about inlining algorithm. */ |
164 static int ncalls_inlined; | 168 static int ncalls_inlined; |
165 static int nfunctions_inlined; | 169 static int nfunctions_inlined; |
166 static int overall_insns; | 170 static int overall_size; |
167 static gcov_type max_count; | 171 static gcov_type max_count, max_benefit; |
168 | 172 |
169 /* Holders of ipa cgraph hooks: */ | 173 /* Holders of ipa cgraph hooks: */ |
170 static struct cgraph_node_hook_list *function_insertion_hook_holder; | 174 static struct cgraph_node_hook_list *function_insertion_hook_holder; |
171 | 175 |
172 static inline struct inline_summary * | 176 static inline struct inline_summary * |
173 inline_summary (struct cgraph_node *node) | 177 inline_summary (struct cgraph_node *node) |
174 { | 178 { |
175 return &node->local.inline_summary; | 179 return &node->local.inline_summary; |
176 } | 180 } |
177 | 181 |
178 /* Estimate size of the function after inlining WHAT into TO. */ | 182 /* Estimate self time of the function after inlining WHAT into TO. */ |
183 | |
184 static int | |
185 cgraph_estimate_time_after_inlining (int frequency, struct cgraph_node *to, | |
186 struct cgraph_node *what) | |
187 { | |
188 gcov_type time = (((gcov_type)what->global.time | |
189 - inline_summary (what)->time_inlining_benefit) | |
190 * frequency + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE | |
191 + to->global.time; | |
192 if (time < 0) | |
193 time = 0; | |
194 if (time > MAX_TIME) | |
195 time = MAX_TIME; | |
196 return time; | |
197 } | |
198 | |
199 /* Estimate self time of the function after inlining WHAT into TO. */ | |
179 | 200 |
180 static int | 201 static int |
181 cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to, | 202 cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to, |
182 struct cgraph_node *what) | 203 struct cgraph_node *what) |
183 { | 204 { |
184 int size; | 205 int size = (what->global.size - inline_summary (what)->size_inlining_benefit) * times + to->global.size; |
185 tree fndecl = what->decl, arg; | |
186 int call_insns = PARAM_VALUE (PARAM_INLINE_CALL_COST); | |
187 | |
188 for (arg = DECL_ARGUMENTS (fndecl); arg; arg = TREE_CHAIN (arg)) | |
189 call_insns += estimate_move_cost (TREE_TYPE (arg)); | |
190 size = (what->global.insns - call_insns) * times + to->global.insns; | |
191 gcc_assert (size >= 0); | 206 gcc_assert (size >= 0); |
192 return size; | 207 return size; |
208 } | |
209 | |
210 /* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest | |
211 by NEST. */ | |
212 | |
213 static void | |
214 update_noncloned_frequencies (struct cgraph_node *node, | |
215 int freq_scale, int nest) | |
216 { | |
217 struct cgraph_edge *e; | |
218 | |
219 /* We do not want to ignore high loop nest after freq drops to 0. */ | |
220 if (!freq_scale) | |
221 freq_scale = 1; | |
222 for (e = node->callees; e; e = e->next_callee) | |
223 { | |
224 e->loop_nest += nest; | |
225 e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; | |
226 if (e->frequency > CGRAPH_FREQ_MAX) | |
227 e->frequency = CGRAPH_FREQ_MAX; | |
228 if (!e->inline_failed) | |
229 update_noncloned_frequencies (e->callee, freq_scale, nest); | |
230 } | |
193 } | 231 } |
194 | 232 |
195 /* E is expected to be an edge being inlined. Clone destination node of | 233 /* E is expected to be an edge being inlined. Clone destination node of |
196 the edge and redirect it to the new clone. | 234 the edge and redirect it to the new clone. |
197 DUPLICATE is used for bookkeeping on whether we are actually creating new | 235 DUPLICATE is used for bookkeeping on whether we are actually creating new |
206 if (duplicate) | 244 if (duplicate) |
207 { | 245 { |
208 /* We may eliminate the need for out-of-line copy to be output. | 246 /* We may eliminate the need for out-of-line copy to be output. |
209 In that case just go ahead and re-use it. */ | 247 In that case just go ahead and re-use it. */ |
210 if (!e->callee->callers->next_caller | 248 if (!e->callee->callers->next_caller |
211 && !e->callee->needed | 249 && cgraph_can_remove_if_no_direct_calls_p (e->callee) |
250 /* Don't reuse if more than one function shares a comdat group. | |
251 If the other function(s) are needed, we need to emit even | |
252 this function out of line. */ | |
253 && !e->callee->same_comdat_group | |
212 && !cgraph_new_nodes) | 254 && !cgraph_new_nodes) |
213 { | 255 { |
214 gcc_assert (!e->callee->global.inlined_to); | 256 gcc_assert (!e->callee->global.inlined_to); |
215 if (e->callee->analyzed) | 257 if (e->callee->analyzed) |
216 overall_insns -= e->callee->global.insns, nfunctions_inlined++; | 258 { |
259 overall_size -= e->callee->global.size; | |
260 nfunctions_inlined++; | |
261 } | |
217 duplicate = false; | 262 duplicate = false; |
263 e->callee->local.externally_visible = false; | |
264 update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest); | |
218 } | 265 } |
219 else | 266 else |
220 { | 267 { |
221 struct cgraph_node *n; | 268 struct cgraph_node *n; |
222 n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest, | 269 n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest, |
223 update_original); | 270 update_original, NULL); |
224 cgraph_redirect_edge_callee (e, n); | 271 cgraph_redirect_edge_callee (e, n); |
225 } | 272 } |
226 } | 273 } |
227 | 274 |
228 if (e->caller->global.inlined_to) | 275 if (e->caller->global.inlined_to) |
251 | 298 |
252 static bool | 299 static bool |
253 cgraph_mark_inline_edge (struct cgraph_edge *e, bool update_original, | 300 cgraph_mark_inline_edge (struct cgraph_edge *e, bool update_original, |
254 VEC (cgraph_edge_p, heap) **new_edges) | 301 VEC (cgraph_edge_p, heap) **new_edges) |
255 { | 302 { |
256 int old_insns = 0, new_insns = 0; | 303 int old_size = 0, new_size = 0; |
257 struct cgraph_node *to = NULL, *what; | 304 struct cgraph_node *to = NULL, *what; |
258 struct cgraph_edge *curr = e; | 305 struct cgraph_edge *curr = e; |
259 | 306 int freq; |
260 if (e->callee->inline_decl) | 307 bool duplicate = false; |
261 cgraph_redirect_edge_callee (e, cgraph_node (e->callee->inline_decl)); | 308 int orig_size = e->callee->global.size; |
262 | 309 |
263 gcc_assert (e->inline_failed); | 310 gcc_assert (e->inline_failed); |
264 e->inline_failed = NULL; | 311 e->inline_failed = CIF_OK; |
265 | 312 |
266 if (!e->callee->global.inlined) | 313 if (!e->callee->global.inlined) |
267 DECL_POSSIBLY_INLINED (e->callee->decl) = true; | 314 DECL_POSSIBLY_INLINED (e->callee->decl) = true; |
268 e->callee->global.inlined = true; | 315 e->callee->global.inlined = true; |
269 | 316 |
317 if (e->callee->callers->next_caller | |
318 || !cgraph_can_remove_if_no_direct_calls_p (e->callee) | |
319 || e->callee->same_comdat_group) | |
320 duplicate = true; | |
270 cgraph_clone_inlined_nodes (e, true, update_original); | 321 cgraph_clone_inlined_nodes (e, true, update_original); |
271 | 322 |
272 what = e->callee; | 323 what = e->callee; |
273 | 324 |
325 freq = e->frequency; | |
274 /* Now update size of caller and all functions caller is inlined into. */ | 326 /* Now update size of caller and all functions caller is inlined into. */ |
275 for (;e && !e->inline_failed; e = e->caller->callers) | 327 for (;e && !e->inline_failed; e = e->caller->callers) |
276 { | 328 { |
277 old_insns = e->caller->global.insns; | |
278 new_insns = cgraph_estimate_size_after_inlining (1, e->caller, | |
279 what); | |
280 gcc_assert (new_insns >= 0); | |
281 to = e->caller; | 329 to = e->caller; |
282 to->global.insns = new_insns; | 330 old_size = e->caller->global.size; |
331 new_size = cgraph_estimate_size_after_inlining (1, to, what); | |
332 to->global.size = new_size; | |
333 to->global.time = cgraph_estimate_time_after_inlining (freq, to, what); | |
283 } | 334 } |
284 gcc_assert (what->global.inlined_to == to); | 335 gcc_assert (what->global.inlined_to == to); |
285 if (new_insns > old_insns) | 336 if (new_size > old_size) |
286 overall_insns += new_insns - old_insns; | 337 overall_size += new_size - old_size; |
338 if (!duplicate) | |
339 overall_size -= orig_size; | |
287 ncalls_inlined++; | 340 ncalls_inlined++; |
288 | 341 |
289 if (flag_indirect_inlining) | 342 if (flag_indirect_inlining) |
290 return ipa_propagate_indirect_call_infos (curr, new_edges); | 343 return ipa_propagate_indirect_call_infos (curr, new_edges); |
291 else | 344 else |
301 { | 354 { |
302 struct cgraph_node *to = edge->caller; | 355 struct cgraph_node *to = edge->caller; |
303 struct cgraph_node *what = edge->callee; | 356 struct cgraph_node *what = edge->callee; |
304 struct cgraph_edge *e, *next; | 357 struct cgraph_edge *e, *next; |
305 | 358 |
306 gcc_assert (!gimple_call_cannot_inline_p (edge->call_stmt)); | 359 gcc_assert (!edge->call_stmt_cannot_inline_p); |
307 /* Look for all calls, mark them inline and clone recursively | 360 /* Look for all calls, mark them inline and clone recursively |
308 all inlined functions. */ | 361 all inlined functions. */ |
309 for (e = what->callers; e; e = next) | 362 for (e = what->callers; e; e = next) |
310 { | 363 { |
311 next = e->next_caller; | 364 next = e->next_caller; |
336 { | 389 { |
337 if (e->caller == node) | 390 if (e->caller == node) |
338 self_recursive = true; | 391 self_recursive = true; |
339 if (e->inline_failed) | 392 if (e->inline_failed) |
340 growth += (cgraph_estimate_size_after_inlining (1, e->caller, node) | 393 growth += (cgraph_estimate_size_after_inlining (1, e->caller, node) |
341 - e->caller->global.insns); | 394 - e->caller->global.size); |
342 } | 395 } |
343 | 396 |
344 /* ??? Wrong for non-trivially self recursive functions or cases where | 397 /* ??? Wrong for non-trivially self recursive functions or cases where |
345 we decide to not inline for different reasons, but it is not big deal | 398 we decide to not inline for different reasons, but it is not big deal |
346 as in that case we will keep the body around, but we will also avoid | 399 as in that case we will keep the body around, but we will also avoid |
347 some inlining. */ | 400 some inlining. */ |
348 if (!node->needed && !DECL_EXTERNAL (node->decl) && !self_recursive) | 401 if (cgraph_only_called_directly_p (node) |
349 growth -= node->global.insns; | 402 && !DECL_EXTERNAL (node->decl) && !self_recursive) |
403 growth -= node->global.size; | |
350 | 404 |
351 node->global.estimated_growth = growth; | 405 node->global.estimated_growth = growth; |
352 return growth; | 406 return growth; |
353 } | 407 } |
354 | 408 |
355 /* Return false when inlining WHAT into TO is not good idea | 409 /* Return false when inlining WHAT into TO is not good idea |
356 as it would cause too large growth of function bodies. | 410 as it would cause too large growth of function bodies. |
357 When ONE_ONLY is true, assume that only one call site is going | 411 When ONE_ONLY is true, assume that only one call site is going |
358 to be inlined, otherwise figure out how many call sites in | 412 to be inlined, otherwise figure out how many call sites in |
359 TO calls WHAT and verify that all can be inlined. | 413 TO calls WHAT and verify that all can be inlined. |
360 */ | 414 */ |
361 | 415 |
362 static bool | 416 static bool |
363 cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what, | 417 cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what, |
364 const char **reason, bool one_only) | 418 cgraph_inline_failed_t *reason, bool one_only) |
365 { | 419 { |
366 int times = 0; | 420 int times = 0; |
367 struct cgraph_edge *e; | 421 struct cgraph_edge *e; |
368 int newsize; | 422 int newsize; |
369 int limit; | 423 int limit; |
379 if (to->global.inlined_to) | 433 if (to->global.inlined_to) |
380 to = to->global.inlined_to; | 434 to = to->global.inlined_to; |
381 | 435 |
382 /* When inlining large function body called once into small function, | 436 /* When inlining large function body called once into small function, |
383 take the inlined function as base for limiting the growth. */ | 437 take the inlined function as base for limiting the growth. */ |
384 if (inline_summary (to)->self_insns > inline_summary(what)->self_insns) | 438 if (inline_summary (to)->self_size > inline_summary(what)->self_size) |
385 limit = inline_summary (to)->self_insns; | 439 limit = inline_summary (to)->self_size; |
386 else | 440 else |
387 limit = inline_summary (what)->self_insns; | 441 limit = inline_summary (what)->self_size; |
388 | 442 |
389 limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100; | 443 limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100; |
390 | 444 |
391 /* Check the size after inlining against the function limits. But allow | 445 /* Check the size after inlining against the function limits. But allow |
392 the function to shrink if it went over the limits by forced inlining. */ | 446 the function to shrink if it went over the limits by forced inlining. */ |
393 newsize = cgraph_estimate_size_after_inlining (times, to, what); | 447 newsize = cgraph_estimate_size_after_inlining (times, to, what); |
394 if (newsize >= to->global.insns | 448 if (newsize >= to->global.size |
395 && newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS) | 449 && newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS) |
396 && newsize > limit) | 450 && newsize > limit) |
397 { | 451 { |
398 if (reason) | 452 if (reason) |
399 *reason = N_("--param large-function-growth limit reached"); | 453 *reason = CIF_LARGE_FUNCTION_GROWTH_LIMIT; |
400 return false; | 454 return false; |
401 } | 455 } |
402 | 456 |
403 stack_size_limit = inline_summary (to)->estimated_self_stack_size; | 457 stack_size_limit = inline_summary (to)->estimated_self_stack_size; |
404 | 458 |
409 + what->global.estimated_stack_size); | 463 + what->global.estimated_stack_size); |
410 if (inlined_stack > stack_size_limit | 464 if (inlined_stack > stack_size_limit |
411 && inlined_stack > PARAM_VALUE (PARAM_LARGE_STACK_FRAME)) | 465 && inlined_stack > PARAM_VALUE (PARAM_LARGE_STACK_FRAME)) |
412 { | 466 { |
413 if (reason) | 467 if (reason) |
414 *reason = N_("--param large-stack-frame-growth limit reached"); | 468 *reason = CIF_LARGE_STACK_FRAME_GROWTH_LIMIT; |
415 return false; | 469 return false; |
416 } | 470 } |
417 return true; | 471 return true; |
418 } | 472 } |
419 | 473 |
420 /* Return true when function N is small enough to be inlined. */ | 474 /* Return true when function N is small enough to be inlined. */ |
421 | 475 |
422 bool | 476 static bool |
423 cgraph_default_inline_p (struct cgraph_node *n, const char **reason) | 477 cgraph_default_inline_p (struct cgraph_node *n, cgraph_inline_failed_t *reason) |
424 { | 478 { |
425 tree decl = n->decl; | 479 tree decl = n->decl; |
426 | 480 |
427 if (n->inline_decl) | |
428 decl = n->inline_decl; | |
429 if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl)) | 481 if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl)) |
430 { | 482 { |
431 if (reason) | 483 if (reason) |
432 *reason = N_("function not inline candidate"); | 484 *reason = CIF_FUNCTION_NOT_INLINE_CANDIDATE; |
433 return false; | 485 return false; |
434 } | 486 } |
435 | 487 |
436 if (!DECL_STRUCT_FUNCTION (decl)->cfg) | 488 if (!n->analyzed) |
437 { | 489 { |
438 if (reason) | 490 if (reason) |
439 *reason = N_("function body not available"); | 491 *reason = CIF_BODY_NOT_AVAILABLE; |
440 return false; | 492 return false; |
441 } | 493 } |
442 | 494 |
443 if (DECL_DECLARED_INLINE_P (decl)) | 495 if (DECL_DECLARED_INLINE_P (decl)) |
444 { | 496 { |
445 if (n->global.insns >= MAX_INLINE_INSNS_SINGLE) | 497 if (n->global.size >= MAX_INLINE_INSNS_SINGLE) |
446 { | 498 { |
447 if (reason) | 499 if (reason) |
448 *reason = N_("--param max-inline-insns-single limit reached"); | 500 *reason = CIF_MAX_INLINE_INSNS_SINGLE_LIMIT; |
449 return false; | 501 return false; |
450 } | 502 } |
451 } | 503 } |
452 else | 504 else |
453 { | 505 { |
454 if (n->global.insns >= MAX_INLINE_INSNS_AUTO) | 506 if (n->global.size >= MAX_INLINE_INSNS_AUTO) |
455 { | 507 { |
456 if (reason) | 508 if (reason) |
457 *reason = N_("--param max-inline-insns-auto limit reached"); | 509 *reason = CIF_MAX_INLINE_INSNS_AUTO_LIMIT; |
458 return false; | 510 return false; |
459 } | 511 } |
460 } | 512 } |
461 | 513 |
462 return true; | 514 return true; |
467 once in the single recursion nest path in the inline graph. */ | 519 once in the single recursion nest path in the inline graph. */ |
468 | 520 |
469 static bool | 521 static bool |
470 cgraph_recursive_inlining_p (struct cgraph_node *to, | 522 cgraph_recursive_inlining_p (struct cgraph_node *to, |
471 struct cgraph_node *what, | 523 struct cgraph_node *what, |
472 const char **reason) | 524 cgraph_inline_failed_t *reason) |
473 { | 525 { |
474 bool recursive; | 526 bool recursive; |
475 if (to->global.inlined_to) | 527 if (to->global.inlined_to) |
476 recursive = what->decl == to->global.inlined_to->decl; | 528 recursive = what->decl == to->global.inlined_to->decl; |
477 else | 529 else |
478 recursive = what->decl == to->decl; | 530 recursive = what->decl == to->decl; |
479 /* Marking recursive function inline has sane semantic and thus we should | 531 /* Marking recursive function inline has sane semantic and thus we should |
480 not warn on it. */ | 532 not warn on it. */ |
481 if (recursive && reason) | 533 if (recursive && reason) |
482 *reason = (what->local.disregard_inline_limits | 534 *reason = (what->local.disregard_inline_limits |
483 ? N_("recursive inlining") : ""); | 535 ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED); |
484 return recursive; | 536 return recursive; |
485 } | 537 } |
486 | 538 |
487 /* A cost model driving the inlining heuristics in a way so the edges with | 539 /* A cost model driving the inlining heuristics in a way so the edges with |
488 smallest badness are inlined first. After each inlining is performed | 540 smallest badness are inlined first. After each inlining is performed |
491 of the function or function body size. */ | 543 of the function or function body size. */ |
492 | 544 |
493 static int | 545 static int |
494 cgraph_edge_badness (struct cgraph_edge *edge) | 546 cgraph_edge_badness (struct cgraph_edge *edge) |
495 { | 547 { |
496 int badness; | 548 gcov_type badness; |
497 int growth = | 549 int growth = |
498 cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee); | 550 cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee); |
499 | 551 |
500 growth -= edge->caller->global.insns; | 552 growth -= edge->caller->global.size; |
501 | 553 |
502 /* Always prefer inlining saving code size. */ | 554 /* Always prefer inlining saving code size. */ |
503 if (growth <= 0) | 555 if (growth <= 0) |
504 badness = INT_MIN - growth; | 556 badness = INT_MIN - growth; |
505 | 557 |
506 /* When profiling is available, base priorities -(#calls / growth). | 558 /* When profiling is available, base priorities -(#calls / growth). |
507 So we optimize for overall number of "executed" inlined calls. */ | 559 So we optimize for overall number of "executed" inlined calls. */ |
508 else if (max_count) | 560 else if (max_count) |
509 badness = ((int)((double)edge->count * INT_MIN / max_count)) / growth; | 561 badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1)) |
562 * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth; | |
510 | 563 |
511 /* When function local profile is available, base priorities on | 564 /* When function local profile is available, base priorities on |
512 growth / frequency, so we optimize for overall frequency of inlined | 565 growth / frequency, so we optimize for overall frequency of inlined |
513 calls. This is not too accurate since while the call might be frequent | 566 calls. This is not too accurate since while the call might be frequent |
514 within function, the function itself is infrequent. | 567 within function, the function itself is infrequent. |
517 We add the estimated growth after inlining all functions to bias the | 570 We add the estimated growth after inlining all functions to bias the |
518 priorities slightly in this direction (so fewer times called functions | 571 priorities slightly in this direction (so fewer times called functions |
519 of the same size gets priority). */ | 572 of the same size gets priority). */ |
520 else if (flag_guess_branch_prob) | 573 else if (flag_guess_branch_prob) |
521 { | 574 { |
522 int div = edge->frequency * 100 / CGRAPH_FREQ_BASE; | 575 int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1; |
523 int growth = | 576 badness = growth * 10000; |
524 cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee); | 577 div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit |
525 growth -= edge->caller->global.insns; | 578 / (edge->callee->global.time + 1) + 1, 100); |
526 badness = growth * 256; | 579 |
527 | 580 |
528 /* Decrease badness if call is nested. */ | 581 /* Decrease badness if call is nested. */ |
529 /* Compress the range so we don't overflow. */ | 582 /* Compress the range so we don't overflow. */ |
530 if (div > 256) | 583 if (div > 10000) |
531 div = 256 + ceil_log2 (div) - 8; | 584 div = 10000 + ceil_log2 (div) - 8; |
532 if (div < 1) | 585 if (div < 1) |
533 div = 1; | 586 div = 1; |
534 if (badness > 0) | 587 if (badness > 0) |
535 badness /= div; | 588 badness /= div; |
536 badness += cgraph_estimate_growth (edge->callee); | 589 badness += cgraph_estimate_growth (edge->callee); |
590 if (badness > INT_MAX) | |
591 badness = INT_MAX; | |
537 } | 592 } |
538 /* When function local profile is not available or it does not give | 593 /* When function local profile is not available or it does not give |
539 useful information (ie frequency is zero), base the cost on | 594 useful information (ie frequency is zero), base the cost on |
540 loop nest and overall size growth, so we optimize for overall number | 595 loop nest and overall size growth, so we optimize for overall number |
541 of functions fully inlined in program. */ | 596 of functions fully inlined in program. */ |
543 { | 598 { |
544 int nest = MIN (edge->loop_nest, 8); | 599 int nest = MIN (edge->loop_nest, 8); |
545 badness = cgraph_estimate_growth (edge->callee) * 256; | 600 badness = cgraph_estimate_growth (edge->callee) * 256; |
546 | 601 |
547 /* Decrease badness if call is nested. */ | 602 /* Decrease badness if call is nested. */ |
548 if (badness > 0) | 603 if (badness > 0) |
549 badness >>= nest; | 604 badness >>= nest; |
550 else | 605 else |
551 { | 606 { |
552 badness <<= nest; | 607 badness <<= nest; |
553 } | 608 } |
564 static void | 619 static void |
565 update_caller_keys (fibheap_t heap, struct cgraph_node *node, | 620 update_caller_keys (fibheap_t heap, struct cgraph_node *node, |
566 bitmap updated_nodes) | 621 bitmap updated_nodes) |
567 { | 622 { |
568 struct cgraph_edge *edge; | 623 struct cgraph_edge *edge; |
569 const char *failed_reason; | 624 cgraph_inline_failed_t failed_reason; |
570 | 625 |
571 if (!node->local.inlinable || node->local.disregard_inline_limits | 626 if (!node->local.inlinable || node->local.disregard_inline_limits |
572 || node->global.inlined_to) | 627 || node->global.inlined_to) |
573 return; | 628 return; |
574 if (bitmap_bit_p (updated_nodes, node->uid)) | 629 if (bitmap_bit_p (updated_nodes, node->uid)) |
692 fibheap_delete (heap); | 747 fibheap_delete (heap); |
693 return false; | 748 return false; |
694 } | 749 } |
695 | 750 |
696 if (dump_file) | 751 if (dump_file) |
697 fprintf (dump_file, | 752 fprintf (dump_file, |
698 " Performing recursive inlining on %s\n", | 753 " Performing recursive inlining on %s\n", |
699 cgraph_node_name (node)); | 754 cgraph_node_name (node)); |
700 | 755 |
701 /* We need original clone to copy around. */ | 756 /* We need original clone to copy around. */ |
702 master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1, false); | 757 master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1, |
758 false, NULL); | |
703 master_clone->needed = true; | 759 master_clone->needed = true; |
704 for (e = master_clone->callees; e; e = e->next_callee) | 760 for (e = master_clone->callees; e; e = e->next_callee) |
705 if (!e->inline_failed) | 761 if (!e->inline_failed) |
706 cgraph_clone_inlined_nodes (e, true, false); | 762 cgraph_clone_inlined_nodes (e, true, false); |
707 | 763 |
720 if (node->decl == curr->callee->decl) | 776 if (node->decl == curr->callee->decl) |
721 depth++; | 777 depth++; |
722 if (depth > max_depth) | 778 if (depth > max_depth) |
723 { | 779 { |
724 if (dump_file) | 780 if (dump_file) |
725 fprintf (dump_file, | 781 fprintf (dump_file, |
726 " maximal depth reached\n"); | 782 " maximal depth reached\n"); |
727 continue; | 783 continue; |
728 } | 784 } |
729 | 785 |
730 if (max_count) | 786 if (max_count) |
736 continue; | 792 continue; |
737 } | 793 } |
738 if (curr->count * 100 / node->count < probability) | 794 if (curr->count * 100 / node->count < probability) |
739 { | 795 { |
740 if (dump_file) | 796 if (dump_file) |
741 fprintf (dump_file, | 797 fprintf (dump_file, |
742 " Probability of edge is too small\n"); | 798 " Probability of edge is too small\n"); |
743 continue; | 799 continue; |
744 } | 800 } |
745 } | 801 } |
746 | 802 |
747 if (dump_file) | 803 if (dump_file) |
748 { | 804 { |
749 fprintf (dump_file, | 805 fprintf (dump_file, |
750 " Inlining call of depth %i", depth); | 806 " Inlining call of depth %i", depth); |
751 if (node->count) | 807 if (node->count) |
752 { | 808 { |
753 fprintf (dump_file, " called approx. %.2f times per call", | 809 fprintf (dump_file, " called approx. %.2f times per call", |
754 (double)curr->count / node->count); | 810 (double)curr->count / node->count); |
763 if (!fibheap_empty (heap) && dump_file) | 819 if (!fibheap_empty (heap) && dump_file) |
764 fprintf (dump_file, " Recursive inlining growth limit met.\n"); | 820 fprintf (dump_file, " Recursive inlining growth limit met.\n"); |
765 | 821 |
766 fibheap_delete (heap); | 822 fibheap_delete (heap); |
767 if (dump_file) | 823 if (dump_file) |
768 fprintf (dump_file, | 824 fprintf (dump_file, |
769 "\n Inlined %i times, body grown from %i to %i insns\n", n, | 825 "\n Inlined %i times, body grown from size %i to %i, time %i to %i\n", n, |
770 master_clone->global.insns, node->global.insns); | 826 master_clone->global.size, node->global.size, |
827 master_clone->global.time, node->global.time); | |
771 | 828 |
772 /* Remove master clone we used for inlining. We rely that clones inlined | 829 /* Remove master clone we used for inlining. We rely that clones inlined |
773 into master clone gets queued just before master clone so we don't | 830 into master clone gets queued just before master clone so we don't |
774 need recursion. */ | 831 need recursion. */ |
775 for (node = cgraph_nodes; node != master_clone; | 832 for (node = cgraph_nodes; node != master_clone; |
788 } | 845 } |
789 | 846 |
790 /* Set inline_failed for all callers of given function to REASON. */ | 847 /* Set inline_failed for all callers of given function to REASON. */ |
791 | 848 |
792 static void | 849 static void |
793 cgraph_set_inline_failed (struct cgraph_node *node, const char *reason) | 850 cgraph_set_inline_failed (struct cgraph_node *node, |
851 cgraph_inline_failed_t reason) | |
794 { | 852 { |
795 struct cgraph_edge *e; | 853 struct cgraph_edge *e; |
796 | 854 |
797 if (dump_file) | 855 if (dump_file) |
798 fprintf (dump_file, "Inlining failed: %s\n", reason); | 856 fprintf (dump_file, "Inlining failed: %s\n", |
857 cgraph_inline_failed_string (reason)); | |
799 for (e = node->callers; e; e = e->next_caller) | 858 for (e = node->callers; e; e = e->next_caller) |
800 if (e->inline_failed) | 859 if (e->inline_failed) |
801 e->inline_failed = reason; | 860 e->inline_failed = reason; |
802 } | 861 } |
803 | 862 |
838 static void | 897 static void |
839 cgraph_decide_inlining_of_small_functions (void) | 898 cgraph_decide_inlining_of_small_functions (void) |
840 { | 899 { |
841 struct cgraph_node *node; | 900 struct cgraph_node *node; |
842 struct cgraph_edge *edge; | 901 struct cgraph_edge *edge; |
843 const char *failed_reason; | 902 cgraph_inline_failed_t failed_reason; |
844 fibheap_t heap = fibheap_new (); | 903 fibheap_t heap = fibheap_new (); |
845 bitmap updated_nodes = BITMAP_ALLOC (NULL); | 904 bitmap updated_nodes = BITMAP_ALLOC (NULL); |
846 int min_insns, max_insns; | 905 int min_size, max_size; |
847 VEC (cgraph_edge_p, heap) *new_indirect_edges = NULL; | 906 VEC (cgraph_edge_p, heap) *new_indirect_edges = NULL; |
848 | 907 |
849 if (flag_indirect_inlining) | 908 if (flag_indirect_inlining) |
850 new_indirect_edges = VEC_alloc (cgraph_edge_p, heap, 8); | 909 new_indirect_edges = VEC_alloc (cgraph_edge_p, heap, 8); |
851 | 910 |
875 gcc_assert (!edge->aux); | 934 gcc_assert (!edge->aux); |
876 edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge); | 935 edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge); |
877 } | 936 } |
878 } | 937 } |
879 | 938 |
880 max_insns = compute_max_insns (overall_insns); | 939 max_size = compute_max_insns (overall_size); |
881 min_insns = overall_insns; | 940 min_size = overall_size; |
882 | 941 |
883 while (overall_insns <= max_insns | 942 while (overall_size <= max_size |
884 && (edge = (struct cgraph_edge *) fibheap_extract_min (heap))) | 943 && (edge = (struct cgraph_edge *) fibheap_extract_min (heap))) |
885 { | 944 { |
886 int old_insns = overall_insns; | 945 int old_size = overall_size; |
887 struct cgraph_node *where; | 946 struct cgraph_node *where; |
888 int growth = | 947 int growth = |
889 cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee); | 948 cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee); |
890 const char *not_good = NULL; | 949 cgraph_inline_failed_t not_good = CIF_OK; |
891 | 950 |
892 growth -= edge->caller->global.insns; | 951 growth -= edge->caller->global.size; |
893 | 952 |
894 if (dump_file) | 953 if (dump_file) |
895 { | 954 { |
896 fprintf (dump_file, | 955 fprintf (dump_file, |
897 "\nConsidering %s with %i insns\n", | 956 "\nConsidering %s with %i size\n", |
898 cgraph_node_name (edge->callee), | 957 cgraph_node_name (edge->callee), |
899 edge->callee->global.insns); | 958 edge->callee->global.size); |
900 fprintf (dump_file, | 959 fprintf (dump_file, |
901 " to be inlined into %s\n" | 960 " to be inlined into %s in %s:%i\n" |
902 " Estimated growth after inlined into all callees is %+i insns.\n" | 961 " Estimated growth after inlined into all callees is %+i insns.\n" |
903 " Estimated badness is %i, frequency %.2f.\n", | 962 " Estimated badness is %i, frequency %.2f.\n", |
904 cgraph_node_name (edge->caller), | 963 cgraph_node_name (edge->caller), |
964 gimple_filename ((const_gimple) edge->call_stmt), | |
965 gimple_lineno ((const_gimple) edge->call_stmt), | |
905 cgraph_estimate_growth (edge->callee), | 966 cgraph_estimate_growth (edge->callee), |
906 cgraph_edge_badness (edge), | 967 cgraph_edge_badness (edge), |
907 edge->frequency / (double)CGRAPH_FREQ_BASE); | 968 edge->frequency / (double)CGRAPH_FREQ_BASE); |
908 if (edge->count) | 969 if (edge->count) |
909 fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count); | 970 fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count); |
937 where = where->callers->caller; | 998 where = where->callers->caller; |
938 } | 999 } |
939 if (where->global.inlined_to) | 1000 if (where->global.inlined_to) |
940 { | 1001 { |
941 edge->inline_failed | 1002 edge->inline_failed |
942 = (edge->callee->local.disregard_inline_limits ? N_("recursive inlining") : ""); | 1003 = (edge->callee->local.disregard_inline_limits |
1004 ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED); | |
943 if (dump_file) | 1005 if (dump_file) |
944 fprintf (dump_file, " inline_failed:Recursive inlining performed only for function itself.\n"); | 1006 fprintf (dump_file, " inline_failed:Recursive inlining performed only for function itself.\n"); |
945 continue; | 1007 continue; |
946 } | 1008 } |
947 } | 1009 } |
948 | 1010 |
949 if (!cgraph_maybe_hot_edge_p (edge)) | 1011 if (!cgraph_maybe_hot_edge_p (edge)) |
950 not_good = N_("call is unlikely and code size would grow"); | 1012 not_good = CIF_UNLIKELY_CALL; |
951 if (!flag_inline_functions | 1013 if (!flag_inline_functions |
952 && !DECL_DECLARED_INLINE_P (edge->callee->decl)) | 1014 && !DECL_DECLARED_INLINE_P (edge->callee->decl)) |
953 not_good = N_("function not declared inline and code size would grow"); | 1015 not_good = CIF_NOT_DECLARED_INLINED; |
954 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl))) | 1016 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl))) |
955 not_good = N_("optimizing for size and code size would grow"); | 1017 not_good = CIF_OPTIMIZING_FOR_SIZE; |
956 if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0) | 1018 if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0) |
957 { | 1019 { |
958 if (!cgraph_recursive_inlining_p (edge->caller, edge->callee, | 1020 if (!cgraph_recursive_inlining_p (edge->caller, edge->callee, |
959 &edge->inline_failed)) | 1021 &edge->inline_failed)) |
960 { | 1022 { |
961 edge->inline_failed = not_good; | 1023 edge->inline_failed = not_good; |
962 if (dump_file) | 1024 if (dump_file) |
963 fprintf (dump_file, " inline_failed:%s.\n", edge->inline_failed); | 1025 fprintf (dump_file, " inline_failed:%s.\n", |
1026 cgraph_inline_failed_string (edge->inline_failed)); | |
964 } | 1027 } |
965 continue; | 1028 continue; |
966 } | 1029 } |
967 if (!cgraph_default_inline_p (edge->callee, &edge->inline_failed)) | 1030 if (!cgraph_default_inline_p (edge->callee, &edge->inline_failed)) |
968 { | 1031 { |
969 if (!cgraph_recursive_inlining_p (edge->caller, edge->callee, | 1032 if (!cgraph_recursive_inlining_p (edge->caller, edge->callee, |
970 &edge->inline_failed)) | 1033 &edge->inline_failed)) |
971 { | 1034 { |
972 if (dump_file) | 1035 if (dump_file) |
973 fprintf (dump_file, " inline_failed:%s.\n", edge->inline_failed); | 1036 fprintf (dump_file, " inline_failed:%s.\n", |
1037 cgraph_inline_failed_string (edge->inline_failed)); | |
974 } | 1038 } |
975 continue; | 1039 continue; |
976 } | 1040 } |
977 if (!tree_can_inline_p (edge->caller->decl, edge->callee->decl)) | 1041 if (!tree_can_inline_p (edge)) |
978 { | 1042 { |
979 gimple_call_set_cannot_inline (edge->call_stmt, true); | |
980 edge->inline_failed = N_("target specific option mismatch"); | |
981 if (dump_file) | 1043 if (dump_file) |
982 fprintf (dump_file, " inline_failed:%s.\n", edge->inline_failed); | 1044 fprintf (dump_file, " inline_failed:%s.\n", |
1045 cgraph_inline_failed_string (edge->inline_failed)); | |
983 continue; | 1046 continue; |
984 } | 1047 } |
985 if (cgraph_recursive_inlining_p (edge->caller, edge->callee, | 1048 if (cgraph_recursive_inlining_p (edge->caller, edge->callee, |
986 &edge->inline_failed)) | 1049 &edge->inline_failed)) |
987 { | 1050 { |
997 update_callee_keys (heap, where, updated_nodes); | 1060 update_callee_keys (heap, where, updated_nodes); |
998 } | 1061 } |
999 else | 1062 else |
1000 { | 1063 { |
1001 struct cgraph_node *callee; | 1064 struct cgraph_node *callee; |
1002 if (gimple_call_cannot_inline_p (edge->call_stmt) | 1065 if (edge->call_stmt_cannot_inline_p |
1003 || !cgraph_check_inline_limits (edge->caller, edge->callee, | 1066 || !cgraph_check_inline_limits (edge->caller, edge->callee, |
1004 &edge->inline_failed, true)) | 1067 &edge->inline_failed, true)) |
1005 { | 1068 { |
1006 if (dump_file) | 1069 if (dump_file) |
1007 fprintf (dump_file, " Not inlining into %s:%s.\n", | 1070 fprintf (dump_file, " Not inlining into %s:%s.\n", |
1008 cgraph_node_name (edge->caller), edge->inline_failed); | 1071 cgraph_node_name (edge->caller), |
1072 cgraph_inline_failed_string (edge->inline_failed)); | |
1009 continue; | 1073 continue; |
1010 } | 1074 } |
1011 callee = edge->callee; | 1075 callee = edge->callee; |
1012 cgraph_mark_inline_edge (edge, true, &new_indirect_edges); | 1076 cgraph_mark_inline_edge (edge, true, &new_indirect_edges); |
1013 if (flag_indirect_inlining) | 1077 if (flag_indirect_inlining) |
1028 update_caller_keys (heap, where, updated_nodes); | 1092 update_caller_keys (heap, where, updated_nodes); |
1029 bitmap_clear (updated_nodes); | 1093 bitmap_clear (updated_nodes); |
1030 | 1094 |
1031 if (dump_file) | 1095 if (dump_file) |
1032 { | 1096 { |
1033 fprintf (dump_file, | 1097 fprintf (dump_file, |
1034 " Inlined into %s which now has %i insns," | 1098 " Inlined into %s which now has size %i and self time %i," |
1035 "net change of %+i insns.\n", | 1099 "net change of %+i.\n", |
1036 cgraph_node_name (edge->caller), | 1100 cgraph_node_name (edge->caller), |
1037 edge->caller->global.insns, | 1101 edge->caller->global.time, |
1038 overall_insns - old_insns); | 1102 edge->caller->global.size, |
1039 } | 1103 overall_size - old_size); |
1040 if (min_insns > overall_insns) | 1104 } |
1041 { | 1105 if (min_size > overall_size) |
1042 min_insns = overall_insns; | 1106 { |
1043 max_insns = compute_max_insns (min_insns); | 1107 min_size = overall_size; |
1108 max_size = compute_max_insns (min_size); | |
1044 | 1109 |
1045 if (dump_file) | 1110 if (dump_file) |
1046 fprintf (dump_file, "New minimal insns reached: %i\n", min_insns); | 1111 fprintf (dump_file, "New minimal size reached: %i\n", min_size); |
1047 } | 1112 } |
1048 } | 1113 } |
1049 while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL) | 1114 while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL) |
1050 { | 1115 { |
1051 gcc_assert (edge->aux); | 1116 gcc_assert (edge->aux); |
1052 edge->aux = NULL; | 1117 edge->aux = NULL; |
1053 if (!edge->callee->local.disregard_inline_limits && edge->inline_failed | 1118 if (!edge->callee->local.disregard_inline_limits && edge->inline_failed |
1054 && !cgraph_recursive_inlining_p (edge->caller, edge->callee, | 1119 && !cgraph_recursive_inlining_p (edge->caller, edge->callee, |
1055 &edge->inline_failed)) | 1120 &edge->inline_failed)) |
1056 edge->inline_failed = N_("--param inline-unit-growth limit reached"); | 1121 edge->inline_failed = CIF_INLINE_UNIT_GROWTH_LIMIT; |
1057 } | 1122 } |
1058 | 1123 |
1059 if (new_indirect_edges) | 1124 if (new_indirect_edges) |
1060 VEC_free (cgraph_edge_p, heap, new_indirect_edges); | 1125 VEC_free (cgraph_edge_p, heap, new_indirect_edges); |
1061 fibheap_delete (heap); | 1126 fibheap_delete (heap); |
1070 { | 1135 { |
1071 struct cgraph_node *node; | 1136 struct cgraph_node *node; |
1072 int nnodes; | 1137 int nnodes; |
1073 struct cgraph_node **order = | 1138 struct cgraph_node **order = |
1074 XCNEWVEC (struct cgraph_node *, cgraph_n_nodes); | 1139 XCNEWVEC (struct cgraph_node *, cgraph_n_nodes); |
1075 int old_insns = 0; | 1140 int old_size = 0; |
1076 int i; | 1141 int i; |
1077 int initial_insns = 0; | |
1078 bool redo_always_inline = true; | 1142 bool redo_always_inline = true; |
1143 int initial_size = 0; | |
1079 | 1144 |
1080 cgraph_remove_function_insertion_hook (function_insertion_hook_holder); | 1145 cgraph_remove_function_insertion_hook (function_insertion_hook_holder); |
1146 if (in_lto_p && flag_indirect_inlining) | |
1147 ipa_update_after_lto_read (); | |
1081 | 1148 |
1082 max_count = 0; | 1149 max_count = 0; |
1150 max_benefit = 0; | |
1083 for (node = cgraph_nodes; node; node = node->next) | 1151 for (node = cgraph_nodes; node; node = node->next) |
1084 if (node->analyzed && (node->needed || node->reachable)) | 1152 if (node->analyzed) |
1085 { | 1153 { |
1086 struct cgraph_edge *e; | 1154 struct cgraph_edge *e; |
1087 | 1155 |
1088 initial_insns += inline_summary (node)->self_insns; | 1156 gcc_assert (inline_summary (node)->self_size == node->global.size); |
1089 gcc_assert (inline_summary (node)->self_insns == node->global.insns); | 1157 initial_size += node->global.size; |
1090 for (e = node->callees; e; e = e->next_callee) | 1158 for (e = node->callees; e; e = e->next_callee) |
1091 if (max_count < e->count) | 1159 if (max_count < e->count) |
1092 max_count = e->count; | 1160 max_count = e->count; |
1161 if (max_benefit < inline_summary (node)->time_inlining_benefit) | |
1162 max_benefit = inline_summary (node)->time_inlining_benefit; | |
1093 } | 1163 } |
1094 overall_insns = initial_insns; | 1164 gcc_assert (in_lto_p |
1095 gcc_assert (!max_count || (profile_info && flag_branch_probabilities)); | 1165 || !max_count |
1166 || (profile_info && flag_branch_probabilities)); | |
1167 overall_size = initial_size; | |
1096 | 1168 |
1097 nnodes = cgraph_postorder (order); | 1169 nnodes = cgraph_postorder (order); |
1098 | 1170 |
1099 if (dump_file) | 1171 if (dump_file) |
1100 fprintf (dump_file, | 1172 fprintf (dump_file, |
1101 "\nDeciding on inlining. Starting with %i insns.\n", | 1173 "\nDeciding on inlining. Starting with size %i.\n", |
1102 initial_insns); | 1174 initial_size); |
1103 | 1175 |
1104 for (node = cgraph_nodes; node; node = node->next) | 1176 for (node = cgraph_nodes; node; node = node->next) |
1105 node->aux = 0; | 1177 node->aux = 0; |
1106 | 1178 |
1107 if (dump_file) | 1179 if (dump_file) |
1131 | 1203 |
1132 if (!node->local.disregard_inline_limits) | 1204 if (!node->local.disregard_inline_limits) |
1133 continue; | 1205 continue; |
1134 if (dump_file) | 1206 if (dump_file) |
1135 fprintf (dump_file, | 1207 fprintf (dump_file, |
1136 "\nConsidering %s %i insns (always inline)\n", | 1208 "\nConsidering %s size:%i (always inline)\n", |
1137 cgraph_node_name (node), node->global.insns); | 1209 cgraph_node_name (node), node->global.size); |
1138 old_insns = overall_insns; | 1210 old_size = overall_size; |
1139 for (e = node->callers; e; e = next) | 1211 for (e = node->callers; e; e = next) |
1140 { | 1212 { |
1141 next = e->next_caller; | 1213 next = e->next_caller; |
1142 if (!e->inline_failed | 1214 if (!e->inline_failed || e->call_stmt_cannot_inline_p) |
1143 || gimple_call_cannot_inline_p (e->call_stmt)) | |
1144 continue; | 1215 continue; |
1145 if (cgraph_recursive_inlining_p (e->caller, e->callee, | 1216 if (cgraph_recursive_inlining_p (e->caller, e->callee, |
1146 &e->inline_failed)) | 1217 &e->inline_failed)) |
1147 continue; | 1218 continue; |
1148 if (!tree_can_inline_p (e->caller->decl, e->callee->decl)) | 1219 if (!tree_can_inline_p (e)) |
1149 { | 1220 continue; |
1150 gimple_call_set_cannot_inline (e->call_stmt, true); | |
1151 continue; | |
1152 } | |
1153 if (cgraph_mark_inline_edge (e, true, NULL)) | 1221 if (cgraph_mark_inline_edge (e, true, NULL)) |
1154 redo_always_inline = true; | 1222 redo_always_inline = true; |
1155 if (dump_file) | 1223 if (dump_file) |
1156 fprintf (dump_file, | 1224 fprintf (dump_file, |
1157 " Inlined into %s which now has %i insns.\n", | 1225 " Inlined into %s which now has size %i.\n", |
1158 cgraph_node_name (e->caller), | 1226 cgraph_node_name (e->caller), |
1159 e->caller->global.insns); | 1227 e->caller->global.size); |
1160 } | 1228 } |
1161 /* Inlining self recursive function might introduce new calls to | 1229 /* Inlining self recursive function might introduce new calls to |
1162 themselves we didn't see in the loop above. Fill in the proper | 1230 themselves we didn't see in the loop above. Fill in the proper |
1163 reason why inline failed. */ | 1231 reason why inline failed. */ |
1164 for (e = node->callers; e; e = e->next_caller) | 1232 for (e = node->callers; e; e = e->next_caller) |
1165 if (e->inline_failed) | 1233 if (e->inline_failed) |
1166 e->inline_failed = N_("recursive inlining"); | 1234 e->inline_failed = CIF_RECURSIVE_INLINING; |
1167 if (dump_file) | 1235 if (dump_file) |
1168 fprintf (dump_file, | 1236 fprintf (dump_file, |
1169 " Inlined for a net change of %+i insns.\n", | 1237 " Inlined for a net change of %+i size.\n", |
1170 overall_insns - old_insns); | 1238 overall_size - old_size); |
1171 } | 1239 } |
1172 } | 1240 } |
1173 | 1241 |
1174 cgraph_decide_inlining_of_small_functions (); | 1242 cgraph_decide_inlining_of_small_functions (); |
1175 | 1243 |
1183 { | 1251 { |
1184 node = order[i]; | 1252 node = order[i]; |
1185 | 1253 |
1186 if (node->callers | 1254 if (node->callers |
1187 && !node->callers->next_caller | 1255 && !node->callers->next_caller |
1188 && !node->needed | 1256 && cgraph_only_called_directly_p (node) |
1189 && node->local.inlinable | 1257 && node->local.inlinable |
1190 && node->callers->inline_failed | 1258 && node->callers->inline_failed |
1191 && !gimple_call_cannot_inline_p (node->callers->call_stmt) | 1259 && node->callers->caller != node |
1260 && node->callers->caller->global.inlined_to != node | |
1261 && !node->callers->call_stmt_cannot_inline_p | |
1192 && !DECL_EXTERNAL (node->decl) | 1262 && !DECL_EXTERNAL (node->decl) |
1193 && !DECL_COMDAT (node->decl)) | 1263 && !DECL_COMDAT (node->decl)) |
1194 { | 1264 { |
1265 cgraph_inline_failed_t reason; | |
1266 old_size = overall_size; | |
1195 if (dump_file) | 1267 if (dump_file) |
1196 { | 1268 { |
1197 fprintf (dump_file, | 1269 fprintf (dump_file, |
1198 "\nConsidering %s %i insns.\n", | 1270 "\nConsidering %s size %i.\n", |
1199 cgraph_node_name (node), node->global.insns); | 1271 cgraph_node_name (node), node->global.size); |
1200 fprintf (dump_file, | 1272 fprintf (dump_file, |
1201 " Called once from %s %i insns.\n", | 1273 " Called once from %s %i insns.\n", |
1202 cgraph_node_name (node->callers->caller), | 1274 cgraph_node_name (node->callers->caller), |
1203 node->callers->caller->global.insns); | 1275 node->callers->caller->global.size); |
1204 } | 1276 } |
1205 | 1277 |
1206 old_insns = overall_insns; | |
1207 | |
1208 if (cgraph_check_inline_limits (node->callers->caller, node, | 1278 if (cgraph_check_inline_limits (node->callers->caller, node, |
1209 NULL, false)) | 1279 &reason, false)) |
1210 { | 1280 { |
1211 cgraph_mark_inline (node->callers); | 1281 cgraph_mark_inline (node->callers); |
1212 if (dump_file) | 1282 if (dump_file) |
1213 fprintf (dump_file, | 1283 fprintf (dump_file, |
1214 " Inlined into %s which now has %i insns" | 1284 " Inlined into %s which now has %i size" |
1215 " for a net change of %+i insns.\n", | 1285 " for a net change of %+i size.\n", |
1216 cgraph_node_name (node->callers->caller), | 1286 cgraph_node_name (node->callers->caller), |
1217 node->callers->caller->global.insns, | 1287 node->callers->caller->global.size, |
1218 overall_insns - old_insns); | 1288 overall_size - old_size); |
1219 } | 1289 } |
1220 else | 1290 else |
1221 { | 1291 { |
1222 if (dump_file) | 1292 if (dump_file) |
1223 fprintf (dump_file, | 1293 fprintf (dump_file, |
1224 " Inline limit reached, not inlined.\n"); | 1294 " Not inlining: %s.\n", |
1295 cgraph_inline_failed_string (reason)); | |
1225 } | 1296 } |
1226 } | 1297 } |
1227 } | 1298 } |
1228 } | 1299 } |
1229 | 1300 |
1232 free_all_ipa_structures_after_iinln (); | 1303 free_all_ipa_structures_after_iinln (); |
1233 | 1304 |
1234 if (dump_file) | 1305 if (dump_file) |
1235 fprintf (dump_file, | 1306 fprintf (dump_file, |
1236 "\nInlined %i calls, eliminated %i functions, " | 1307 "\nInlined %i calls, eliminated %i functions, " |
1237 "%i insns turned to %i insns.\n\n", | 1308 "size %i turned to %i size.\n\n", |
1238 ncalls_inlined, nfunctions_inlined, initial_insns, | 1309 ncalls_inlined, nfunctions_inlined, initial_size, |
1239 overall_insns); | 1310 overall_size); |
1240 free (order); | 1311 free (order); |
1241 return 0; | 1312 return 0; |
1242 } | 1313 } |
1243 | 1314 |
1244 /* Try to inline edge E from incremental inliner. MODE specifies mode | 1315 /* Try to inline edge E from incremental inliner. MODE specifies mode |
1258 try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth) | 1329 try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth) |
1259 { | 1330 { |
1260 struct cgraph_node *callee = e->callee; | 1331 struct cgraph_node *callee = e->callee; |
1261 enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux; | 1332 enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux; |
1262 bool always_inline = e->callee->local.disregard_inline_limits; | 1333 bool always_inline = e->callee->local.disregard_inline_limits; |
1334 bool inlined = false; | |
1263 | 1335 |
1264 /* We've hit cycle? */ | 1336 /* We've hit cycle? */ |
1265 if (callee_mode) | 1337 if (callee_mode) |
1266 { | 1338 { |
1267 /* It is first time we see it and we are not in ALWAY_INLINE only | 1339 /* It is first time we see it and we are not in ALWAY_INLINE only |
1287 "Not inlining %s into %s to avoid cycle.\n", | 1359 "Not inlining %s into %s to avoid cycle.\n", |
1288 cgraph_node_name (callee), | 1360 cgraph_node_name (callee), |
1289 cgraph_node_name (e->caller)); | 1361 cgraph_node_name (e->caller)); |
1290 } | 1362 } |
1291 e->inline_failed = (e->callee->local.disregard_inline_limits | 1363 e->inline_failed = (e->callee->local.disregard_inline_limits |
1292 ? N_("recursive inlining") : ""); | 1364 ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED); |
1293 return false; | 1365 return false; |
1294 } | 1366 } |
1295 } | 1367 } |
1296 | 1368 |
1297 callee->aux = (void *)(size_t) mode; | 1369 callee->aux = (void *)(size_t) mode; |
1298 if (dump_file) | 1370 if (dump_file) |
1299 { | 1371 { |
1300 indent_to (dump_file, depth); | 1372 indent_to (dump_file, depth); |
1301 fprintf (dump_file, " Inlining %s into %s.\n", | 1373 fprintf (dump_file, " Inlining %s into %s.\n", |
1306 { | 1378 { |
1307 cgraph_mark_inline (e); | 1379 cgraph_mark_inline (e); |
1308 | 1380 |
1309 /* In order to fully inline always_inline functions, we need to | 1381 /* In order to fully inline always_inline functions, we need to |
1310 recurse here, since the inlined functions might not be processed by | 1382 recurse here, since the inlined functions might not be processed by |
1311 incremental inlining at all yet. | 1383 incremental inlining at all yet. |
1312 | 1384 |
1313 Also flattening needs to be done recursively. */ | 1385 Also flattening needs to be done recursively. */ |
1314 | 1386 |
1315 if (mode == INLINE_ALL || always_inline) | 1387 if (mode == INLINE_ALL || always_inline) |
1316 cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1); | 1388 cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1); |
1389 inlined = true; | |
1317 } | 1390 } |
1318 callee->aux = (void *)(size_t) callee_mode; | 1391 callee->aux = (void *)(size_t) callee_mode; |
1392 return inlined; | |
1393 } | |
1394 | |
1395 /* Return true when N is leaf function. Accept cheap (pure&const) builtins | |
1396 in leaf functions. */ | |
1397 static bool | |
1398 leaf_node_p (struct cgraph_node *n) | |
1399 { | |
1400 struct cgraph_edge *e; | |
1401 for (e = n->callees; e; e = e->next_callee) | |
1402 if (!DECL_BUILT_IN (e->callee->decl) | |
1403 || (!TREE_READONLY (e->callee->decl) | |
1404 || DECL_PURE_P (e->callee->decl))) | |
1405 return false; | |
1319 return true; | 1406 return true; |
1320 } | 1407 } |
1321 | 1408 |
1322 /* Decide on the inlining. We do so in the topological order to avoid | 1409 /* Decide on the inlining. We do so in the topological order to avoid |
1323 expenses on updating data structures. | 1410 expenses on updating data structures. |
1324 DEPTH is depth of recursion, used only for debug output. */ | 1411 DEPTH is depth of recursion, used only for debug output. */ |
1325 | 1412 |
1326 static bool | 1413 static bool |
1327 cgraph_decide_inlining_incrementally (struct cgraph_node *node, | 1414 cgraph_decide_inlining_incrementally (struct cgraph_node *node, |
1328 enum inlining_mode mode, | 1415 enum inlining_mode mode, |
1329 int depth) | 1416 int depth) |
1330 { | 1417 { |
1331 struct cgraph_edge *e; | 1418 struct cgraph_edge *e; |
1332 bool inlined = false; | 1419 bool inlined = false; |
1333 const char *failed_reason; | 1420 cgraph_inline_failed_t failed_reason; |
1334 enum inlining_mode old_mode; | 1421 enum inlining_mode old_mode; |
1335 | 1422 |
1336 #ifdef ENABLE_CHECKING | 1423 #ifdef ENABLE_CHECKING |
1337 verify_cgraph_node (node); | 1424 verify_cgraph_node (node); |
1338 #endif | 1425 #endif |
1339 | 1426 |
1340 old_mode = (enum inlining_mode) (size_t)node->aux; | 1427 old_mode = (enum inlining_mode) (size_t)node->aux; |
1341 | 1428 |
1342 if (mode != INLINE_ALWAYS_INLINE | 1429 if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE |
1343 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL) | 1430 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL) |
1344 { | 1431 { |
1345 if (dump_file) | 1432 if (dump_file) |
1346 { | 1433 { |
1347 indent_to (dump_file, depth); | 1434 indent_to (dump_file, depth); |
1351 } | 1438 } |
1352 | 1439 |
1353 node->aux = (void *)(size_t) mode; | 1440 node->aux = (void *)(size_t) mode; |
1354 | 1441 |
1355 /* First of all look for always inline functions. */ | 1442 /* First of all look for always inline functions. */ |
1356 for (e = node->callees; e; e = e->next_callee) | 1443 if (mode != INLINE_SIZE_NORECURSIVE) |
1357 { | 1444 for (e = node->callees; e; e = e->next_callee) |
1358 if (!e->callee->local.disregard_inline_limits | 1445 { |
1359 && (mode != INLINE_ALL || !e->callee->local.inlinable)) | 1446 if (!e->callee->local.disregard_inline_limits |
1360 continue; | 1447 && (mode != INLINE_ALL || !e->callee->local.inlinable)) |
1361 if (gimple_call_cannot_inline_p (e->call_stmt)) | |
1362 continue; | |
1363 /* When the edge is already inlined, we just need to recurse into | |
1364 it in order to fully flatten the leaves. */ | |
1365 if (!e->inline_failed && mode == INLINE_ALL) | |
1366 { | |
1367 inlined |= try_inline (e, mode, depth); | |
1368 continue; | 1448 continue; |
1369 } | 1449 if (e->call_stmt_cannot_inline_p) |
1370 if (dump_file) | |
1371 { | |
1372 indent_to (dump_file, depth); | |
1373 fprintf (dump_file, | |
1374 "Considering to always inline inline candidate %s.\n", | |
1375 cgraph_node_name (e->callee)); | |
1376 } | |
1377 if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed)) | |
1378 { | |
1379 if (dump_file) | |
1380 { | |
1381 indent_to (dump_file, depth); | |
1382 fprintf (dump_file, "Not inlining: recursive call.\n"); | |
1383 } | |
1384 continue; | 1450 continue; |
1385 } | 1451 /* When the edge is already inlined, we just need to recurse into |
1386 if (!tree_can_inline_p (node->decl, e->callee->decl)) | 1452 it in order to fully flatten the leaves. */ |
1387 { | 1453 if (!e->inline_failed && mode == INLINE_ALL) |
1388 gimple_call_set_cannot_inline (e->call_stmt, true); | 1454 { |
1389 if (dump_file) | 1455 inlined |= try_inline (e, mode, depth); |
1390 { | 1456 continue; |
1391 indent_to (dump_file, depth); | 1457 } |
1392 fprintf (dump_file, | 1458 if (dump_file) |
1393 "Not inlining: Target specific option mismatch.\n"); | 1459 { |
1394 } | 1460 indent_to (dump_file, depth); |
1395 continue; | 1461 fprintf (dump_file, |
1396 } | 1462 "Considering to always inline inline candidate %s.\n", |
1397 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl)) | 1463 cgraph_node_name (e->callee)); |
1398 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl))) | 1464 } |
1399 { | 1465 if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed)) |
1400 if (dump_file) | 1466 { |
1401 { | 1467 if (dump_file) |
1402 indent_to (dump_file, depth); | 1468 { |
1403 fprintf (dump_file, "Not inlining: SSA form does not match.\n"); | 1469 indent_to (dump_file, depth); |
1404 } | 1470 fprintf (dump_file, "Not inlining: recursive call.\n"); |
1405 continue; | 1471 } |
1406 } | 1472 continue; |
1407 if (!e->callee->analyzed && !e->callee->inline_decl) | 1473 } |
1408 { | 1474 if (!tree_can_inline_p (e)) |
1409 if (dump_file) | 1475 { |
1410 { | 1476 if (dump_file) |
1411 indent_to (dump_file, depth); | 1477 { |
1412 fprintf (dump_file, | 1478 indent_to (dump_file, depth); |
1413 "Not inlining: Function body no longer available.\n"); | 1479 fprintf (dump_file, |
1414 } | 1480 "Not inlining: %s", |
1415 continue; | 1481 cgraph_inline_failed_string (e->inline_failed)); |
1416 } | 1482 } |
1417 inlined |= try_inline (e, mode, depth); | 1483 continue; |
1418 } | 1484 } |
1485 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl)) | |
1486 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl))) | |
1487 { | |
1488 if (dump_file) | |
1489 { | |
1490 indent_to (dump_file, depth); | |
1491 fprintf (dump_file, "Not inlining: SSA form does not match.\n"); | |
1492 } | |
1493 continue; | |
1494 } | |
1495 if (!e->callee->analyzed) | |
1496 { | |
1497 if (dump_file) | |
1498 { | |
1499 indent_to (dump_file, depth); | |
1500 fprintf (dump_file, | |
1501 "Not inlining: Function body no longer available.\n"); | |
1502 } | |
1503 continue; | |
1504 } | |
1505 inlined |= try_inline (e, mode, depth); | |
1506 } | |
1419 | 1507 |
1420 /* Now do the automatic inlining. */ | 1508 /* Now do the automatic inlining. */ |
1421 if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE) | 1509 if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE) |
1422 for (e = node->callees; e; e = e->next_callee) | 1510 for (e = node->callees; e; e = e->next_callee) |
1423 { | 1511 { |
1512 int allowed_growth = 0; | |
1424 if (!e->callee->local.inlinable | 1513 if (!e->callee->local.inlinable |
1425 || !e->inline_failed | 1514 || !e->inline_failed |
1426 || e->callee->local.disregard_inline_limits) | 1515 || e->callee->local.disregard_inline_limits) |
1427 continue; | 1516 continue; |
1428 if (dump_file) | 1517 if (dump_file) |
1445 indent_to (dump_file, depth); | 1534 indent_to (dump_file, depth); |
1446 fprintf (dump_file, "Not inlining: SSA form does not match.\n"); | 1535 fprintf (dump_file, "Not inlining: SSA form does not match.\n"); |
1447 } | 1536 } |
1448 continue; | 1537 continue; |
1449 } | 1538 } |
1539 | |
1540 if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee) | |
1541 && optimize_function_for_speed_p (cfun)) | |
1542 allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS); | |
1543 | |
1450 /* When the function body would grow and inlining the function won't | 1544 /* When the function body would grow and inlining the function won't |
1451 eliminate the need for offline copy of the function, don't inline. | 1545 eliminate the need for offline copy of the function, don't inline. |
1452 */ | 1546 */ |
1453 if ((mode == INLINE_SIZE | 1547 if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE) |
1454 || (!flag_inline_functions | 1548 || (!flag_inline_functions |
1455 && !DECL_DECLARED_INLINE_P (e->callee->decl))) | 1549 && !DECL_DECLARED_INLINE_P (e->callee->decl))) |
1456 && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee) | 1550 && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee) |
1457 > e->caller->global.insns) | 1551 > e->caller->global.size + allowed_growth) |
1458 && cgraph_estimate_growth (e->callee) > 0) | 1552 && cgraph_estimate_growth (e->callee) > allowed_growth) |
1459 { | 1553 { |
1460 if (dump_file) | 1554 if (dump_file) |
1461 { | 1555 { |
1462 indent_to (dump_file, depth); | 1556 indent_to (dump_file, depth); |
1463 fprintf (dump_file, | 1557 fprintf (dump_file, |
1464 "Not inlining: code size would grow by %i insns.\n", | 1558 "Not inlining: code size would grow by %i.\n", |
1465 cgraph_estimate_size_after_inlining (1, e->caller, | 1559 cgraph_estimate_size_after_inlining (1, e->caller, |
1466 e->callee) | 1560 e->callee) |
1467 - e->caller->global.insns); | 1561 - e->caller->global.size); |
1468 } | 1562 } |
1469 continue; | 1563 continue; |
1470 } | 1564 } |
1471 if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed, | 1565 if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed, |
1472 false) | 1566 false) |
1473 || gimple_call_cannot_inline_p (e->call_stmt)) | 1567 || e->call_stmt_cannot_inline_p) |
1474 { | 1568 { |
1475 if (dump_file) | 1569 if (dump_file) |
1476 { | 1570 { |
1477 indent_to (dump_file, depth); | 1571 indent_to (dump_file, depth); |
1478 fprintf (dump_file, "Not inlining: %s.\n", e->inline_failed); | 1572 fprintf (dump_file, "Not inlining: %s.\n", |
1573 cgraph_inline_failed_string (e->inline_failed)); | |
1479 } | 1574 } |
1480 continue; | 1575 continue; |
1481 } | 1576 } |
1482 if (!e->callee->analyzed && !e->callee->inline_decl) | 1577 if (!e->callee->analyzed) |
1483 { | 1578 { |
1484 if (dump_file) | 1579 if (dump_file) |
1485 { | 1580 { |
1486 indent_to (dump_file, depth); | 1581 indent_to (dump_file, depth); |
1487 fprintf (dump_file, | 1582 fprintf (dump_file, |
1488 "Not inlining: Function body no longer available.\n"); | 1583 "Not inlining: Function body no longer available.\n"); |
1489 } | 1584 } |
1490 continue; | 1585 continue; |
1491 } | 1586 } |
1492 if (!tree_can_inline_p (node->decl, e->callee->decl)) | 1587 if (!tree_can_inline_p (e)) |
1493 { | 1588 { |
1494 gimple_call_set_cannot_inline (e->call_stmt, true); | |
1495 if (dump_file) | 1589 if (dump_file) |
1496 { | 1590 { |
1497 indent_to (dump_file, depth); | 1591 indent_to (dump_file, depth); |
1498 fprintf (dump_file, | 1592 fprintf (dump_file, |
1499 "Not inlining: Target specific option mismatch.\n"); | 1593 "Not inlining: %s.", |
1594 cgraph_inline_failed_string (e->inline_failed)); | |
1500 } | 1595 } |
1501 continue; | 1596 continue; |
1502 } | 1597 } |
1503 if (cgraph_default_inline_p (e->callee, &failed_reason)) | 1598 if (cgraph_default_inline_p (e->callee, &failed_reason)) |
1504 inlined |= try_inline (e, mode, depth); | 1599 inlined |= try_inline (e, mode, depth); |
1519 static unsigned int | 1614 static unsigned int |
1520 cgraph_early_inlining (void) | 1615 cgraph_early_inlining (void) |
1521 { | 1616 { |
1522 struct cgraph_node *node = cgraph_node (current_function_decl); | 1617 struct cgraph_node *node = cgraph_node (current_function_decl); |
1523 unsigned int todo = 0; | 1618 unsigned int todo = 0; |
1619 int iterations = 0; | |
1524 | 1620 |
1525 if (sorrycount || errorcount) | 1621 if (sorrycount || errorcount) |
1526 return 0; | 1622 return 0; |
1527 if (cgraph_decide_inlining_incrementally (node, INLINE_SIZE, 0)) | 1623 while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS) |
1624 && cgraph_decide_inlining_incrementally (node, | |
1625 iterations | |
1626 ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0)) | |
1528 { | 1627 { |
1529 timevar_push (TV_INTEGRATION); | 1628 timevar_push (TV_INTEGRATION); |
1530 todo = optimize_inline_calls (current_function_decl); | 1629 todo |= optimize_inline_calls (current_function_decl); |
1630 iterations++; | |
1531 timevar_pop (TV_INTEGRATION); | 1631 timevar_pop (TV_INTEGRATION); |
1532 } | 1632 } |
1633 if (dump_file) | |
1634 fprintf (dump_file, "Iterations: %i\n", iterations); | |
1533 cfun->always_inline_functions_inlined = true; | 1635 cfun->always_inline_functions_inlined = true; |
1534 return todo; | 1636 return todo; |
1535 } | 1637 } |
1536 | 1638 |
1537 /* When inlining shall be performed. */ | 1639 /* When inlining shall be performed. */ |
1539 cgraph_gate_early_inlining (void) | 1641 cgraph_gate_early_inlining (void) |
1540 { | 1642 { |
1541 return flag_early_inlining; | 1643 return flag_early_inlining; |
1542 } | 1644 } |
1543 | 1645 |
1544 struct gimple_opt_pass pass_early_inline = | 1646 struct gimple_opt_pass pass_early_inline = |
1545 { | 1647 { |
1546 { | 1648 { |
1547 GIMPLE_PASS, | 1649 GIMPLE_PASS, |
1548 "einline", /* name */ | 1650 "einline", /* name */ |
1549 cgraph_gate_early_inlining, /* gate */ | 1651 cgraph_gate_early_inlining, /* gate */ |
1551 NULL, /* sub */ | 1653 NULL, /* sub */ |
1552 NULL, /* next */ | 1654 NULL, /* next */ |
1553 0, /* static_pass_number */ | 1655 0, /* static_pass_number */ |
1554 TV_INLINE_HEURISTICS, /* tv_id */ | 1656 TV_INLINE_HEURISTICS, /* tv_id */ |
1555 0, /* properties_required */ | 1657 0, /* properties_required */ |
1556 PROP_cfg, /* properties_provided */ | 1658 0, /* properties_provided */ |
1557 0, /* properties_destroyed */ | 1659 0, /* properties_destroyed */ |
1558 0, /* todo_flags_start */ | 1660 0, /* todo_flags_start */ |
1559 TODO_dump_func /* todo_flags_finish */ | 1661 TODO_dump_func /* todo_flags_finish */ |
1560 } | 1662 } |
1561 }; | 1663 }; |
1563 /* When inlining shall be performed. */ | 1665 /* When inlining shall be performed. */ |
1564 static bool | 1666 static bool |
1565 cgraph_gate_ipa_early_inlining (void) | 1667 cgraph_gate_ipa_early_inlining (void) |
1566 { | 1668 { |
1567 return (flag_early_inlining | 1669 return (flag_early_inlining |
1670 && !in_lto_p | |
1568 && (flag_branch_probabilities || flag_test_coverage | 1671 && (flag_branch_probabilities || flag_test_coverage |
1569 || profile_arc_flag)); | 1672 || profile_arc_flag)); |
1570 } | 1673 } |
1571 | 1674 |
1572 /* IPA pass wrapper for early inlining pass. We need to run early inlining | 1675 /* IPA pass wrapper for early inlining pass. We need to run early inlining |
1573 before tree profiling so we have stand alone IPA pass for doing so. */ | 1676 before tree profiling so we have stand alone IPA pass for doing so. */ |
1574 struct simple_ipa_opt_pass pass_ipa_early_inline = | 1677 struct simple_ipa_opt_pass pass_ipa_early_inline = |
1575 { | 1678 { |
1576 { | 1679 { |
1577 SIMPLE_IPA_PASS, | 1680 SIMPLE_IPA_PASS, |
1578 "einline_ipa", /* name */ | 1681 "einline_ipa", /* name */ |
1579 cgraph_gate_ipa_early_inlining, /* gate */ | 1682 cgraph_gate_ipa_early_inlining, /* gate */ |
1581 NULL, /* sub */ | 1684 NULL, /* sub */ |
1582 NULL, /* next */ | 1685 NULL, /* next */ |
1583 0, /* static_pass_number */ | 1686 0, /* static_pass_number */ |
1584 TV_INLINE_HEURISTICS, /* tv_id */ | 1687 TV_INLINE_HEURISTICS, /* tv_id */ |
1585 0, /* properties_required */ | 1688 0, /* properties_required */ |
1586 PROP_cfg, /* properties_provided */ | 1689 0, /* properties_provided */ |
1587 0, /* properties_destroyed */ | 1690 0, /* properties_destroyed */ |
1588 0, /* todo_flags_start */ | 1691 0, /* todo_flags_start */ |
1589 TODO_dump_cgraph /* todo_flags_finish */ | 1692 TODO_dump_cgraph /* todo_flags_finish */ |
1590 } | 1693 } |
1591 }; | 1694 }; |
1695 | |
1696 /* See if statement might disappear after inlining. We are not terribly | |
1697 sophisficated, basically looking for simple abstraction penalty wrappers. */ | |
1698 | |
1699 static bool | |
1700 likely_eliminated_by_inlining_p (gimple stmt) | |
1701 { | |
1702 enum gimple_code code = gimple_code (stmt); | |
1703 switch (code) | |
1704 { | |
1705 case GIMPLE_RETURN: | |
1706 return true; | |
1707 case GIMPLE_ASSIGN: | |
1708 if (gimple_num_ops (stmt) != 2) | |
1709 return false; | |
1710 | |
1711 /* Casts of parameters, loads from parameters passed by reference | |
1712 and stores to return value or parameters are probably free after | |
1713 inlining. */ | |
1714 if (gimple_assign_rhs_code (stmt) == CONVERT_EXPR | |
1715 || gimple_assign_rhs_code (stmt) == NOP_EXPR | |
1716 || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR | |
1717 || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS) | |
1718 { | |
1719 tree rhs = gimple_assign_rhs1 (stmt); | |
1720 tree lhs = gimple_assign_lhs (stmt); | |
1721 tree inner_rhs = rhs; | |
1722 tree inner_lhs = lhs; | |
1723 bool rhs_free = false; | |
1724 bool lhs_free = false; | |
1725 | |
1726 while (handled_component_p (inner_lhs) || TREE_CODE (inner_lhs) == INDIRECT_REF) | |
1727 inner_lhs = TREE_OPERAND (inner_lhs, 0); | |
1728 while (handled_component_p (inner_rhs) | |
1729 || TREE_CODE (inner_rhs) == ADDR_EXPR || TREE_CODE (inner_rhs) == INDIRECT_REF) | |
1730 inner_rhs = TREE_OPERAND (inner_rhs, 0); | |
1731 | |
1732 | |
1733 if (TREE_CODE (inner_rhs) == PARM_DECL | |
1734 || (TREE_CODE (inner_rhs) == SSA_NAME | |
1735 && SSA_NAME_IS_DEFAULT_DEF (inner_rhs) | |
1736 && TREE_CODE (SSA_NAME_VAR (inner_rhs)) == PARM_DECL)) | |
1737 rhs_free = true; | |
1738 if (rhs_free && is_gimple_reg (lhs)) | |
1739 lhs_free = true; | |
1740 if (((TREE_CODE (inner_lhs) == PARM_DECL | |
1741 || (TREE_CODE (inner_lhs) == SSA_NAME | |
1742 && SSA_NAME_IS_DEFAULT_DEF (inner_lhs) | |
1743 && TREE_CODE (SSA_NAME_VAR (inner_lhs)) == PARM_DECL)) | |
1744 && inner_lhs != lhs) | |
1745 || TREE_CODE (inner_lhs) == RESULT_DECL | |
1746 || (TREE_CODE (inner_lhs) == SSA_NAME | |
1747 && TREE_CODE (SSA_NAME_VAR (inner_lhs)) == RESULT_DECL)) | |
1748 lhs_free = true; | |
1749 if (lhs_free && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs))) | |
1750 rhs_free = true; | |
1751 if (lhs_free && rhs_free) | |
1752 return true; | |
1753 } | |
1754 return false; | |
1755 default: | |
1756 return false; | |
1757 } | |
1758 } | |
1759 | |
1760 /* Compute function body size parameters for NODE. */ | |
1761 | |
1762 static void | |
1763 estimate_function_body_sizes (struct cgraph_node *node) | |
1764 { | |
1765 gcov_type time = 0; | |
1766 gcov_type time_inlining_benefit = 0; | |
1767 int size = 0; | |
1768 int size_inlining_benefit = 0; | |
1769 basic_block bb; | |
1770 gimple_stmt_iterator bsi; | |
1771 struct function *my_function = DECL_STRUCT_FUNCTION (node->decl); | |
1772 tree arg; | |
1773 int freq; | |
1774 tree funtype = TREE_TYPE (node->decl); | |
1775 | |
1776 if (dump_file) | |
1777 fprintf (dump_file, "Analyzing function body size: %s\n", | |
1778 cgraph_node_name (node)); | |
1779 | |
1780 gcc_assert (my_function && my_function->cfg); | |
1781 FOR_EACH_BB_FN (bb, my_function) | |
1782 { | |
1783 freq = compute_call_stmt_bb_frequency (node->decl, bb); | |
1784 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | |
1785 { | |
1786 gimple stmt = gsi_stmt (bsi); | |
1787 int this_size = estimate_num_insns (stmt, &eni_size_weights); | |
1788 int this_time = estimate_num_insns (stmt, &eni_time_weights); | |
1789 | |
1790 if (dump_file && (dump_flags & TDF_DETAILS)) | |
1791 { | |
1792 fprintf (dump_file, " freq:%6i size:%3i time:%3i ", | |
1793 freq, this_size, this_time); | |
1794 print_gimple_stmt (dump_file, stmt, 0, 0); | |
1795 } | |
1796 this_time *= freq; | |
1797 time += this_time; | |
1798 size += this_size; | |
1799 if (likely_eliminated_by_inlining_p (stmt)) | |
1800 { | |
1801 size_inlining_benefit += this_size; | |
1802 time_inlining_benefit += this_time; | |
1803 if (dump_file && (dump_flags & TDF_DETAILS)) | |
1804 fprintf (dump_file, " Likely eliminated\n"); | |
1805 } | |
1806 gcc_assert (time >= 0); | |
1807 gcc_assert (size >= 0); | |
1808 } | |
1809 } | |
1810 time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE; | |
1811 time_inlining_benefit = ((time_inlining_benefit + CGRAPH_FREQ_BASE / 2) | |
1812 / CGRAPH_FREQ_BASE); | |
1813 if (dump_file) | |
1814 fprintf (dump_file, "Overall function body time: %i-%i size: %i-%i\n", | |
1815 (int)time, (int)time_inlining_benefit, | |
1816 size, size_inlining_benefit); | |
1817 time_inlining_benefit += eni_time_weights.call_cost; | |
1818 size_inlining_benefit += eni_size_weights.call_cost; | |
1819 if (!VOID_TYPE_P (TREE_TYPE (funtype))) | |
1820 { | |
1821 int cost = estimate_move_cost (TREE_TYPE (funtype)); | |
1822 time_inlining_benefit += cost; | |
1823 size_inlining_benefit += cost; | |
1824 } | |
1825 for (arg = DECL_ARGUMENTS (node->decl); arg; arg = TREE_CHAIN (arg)) | |
1826 if (!VOID_TYPE_P (TREE_TYPE (arg))) | |
1827 { | |
1828 int cost = estimate_move_cost (TREE_TYPE (arg)); | |
1829 time_inlining_benefit += cost; | |
1830 size_inlining_benefit += cost; | |
1831 } | |
1832 if (time_inlining_benefit > MAX_TIME) | |
1833 time_inlining_benefit = MAX_TIME; | |
1834 if (time > MAX_TIME) | |
1835 time = MAX_TIME; | |
1836 inline_summary (node)->self_time = time; | |
1837 inline_summary (node)->self_size = size; | |
1838 if (dump_file) | |
1839 fprintf (dump_file, "With function call overhead time: %i-%i size: %i-%i\n", | |
1840 (int)time, (int)time_inlining_benefit, | |
1841 size, size_inlining_benefit); | |
1842 inline_summary (node)->time_inlining_benefit = time_inlining_benefit; | |
1843 inline_summary (node)->size_inlining_benefit = size_inlining_benefit; | |
1844 } | |
1592 | 1845 |
1593 /* Compute parameters of functions used by inliner. */ | 1846 /* Compute parameters of functions used by inliner. */ |
1594 unsigned int | 1847 unsigned int |
1595 compute_inline_parameters (struct cgraph_node *node) | 1848 compute_inline_parameters (struct cgraph_node *node) |
1596 { | 1849 { |
1605 node->global.estimated_stack_size = self_stack_size; | 1858 node->global.estimated_stack_size = self_stack_size; |
1606 node->global.stack_frame_offset = 0; | 1859 node->global.stack_frame_offset = 0; |
1607 | 1860 |
1608 /* Can this function be inlined at all? */ | 1861 /* Can this function be inlined at all? */ |
1609 node->local.inlinable = tree_inlinable_function_p (current_function_decl); | 1862 node->local.inlinable = tree_inlinable_function_p (current_function_decl); |
1610 | |
1611 /* Estimate the number of instructions for this function. | |
1612 ??? At -O0 we don't use this information except for the dumps, and | |
1613 even then only for always_inline functions. But disabling this | |
1614 causes ICEs in the inline heuristics... */ | |
1615 inline_summary (node)->self_insns | |
1616 = estimate_num_insns_fn (current_function_decl, &eni_inlining_weights); | |
1617 if (node->local.inlinable && !node->local.disregard_inline_limits) | 1863 if (node->local.inlinable && !node->local.disregard_inline_limits) |
1618 node->local.disregard_inline_limits | 1864 node->local.disregard_inline_limits |
1619 = DECL_DISREGARD_INLINE_LIMITS (current_function_decl); | 1865 = DECL_DISREGARD_INLINE_LIMITS (current_function_decl); |
1620 | 1866 estimate_function_body_sizes (node); |
1621 /* Inlining characteristics are maintained by the cgraph_mark_inline. */ | 1867 /* Inlining characteristics are maintained by the cgraph_mark_inline. */ |
1622 node->global.insns = inline_summary (node)->self_insns; | 1868 node->global.time = inline_summary (node)->self_time; |
1869 node->global.size = inline_summary (node)->self_size; | |
1623 return 0; | 1870 return 0; |
1624 } | 1871 } |
1625 | 1872 |
1626 | 1873 |
1627 /* Compute parameters of functions used by inliner using | 1874 /* Compute parameters of functions used by inliner using |
1631 { | 1878 { |
1632 compute_inline_parameters (cgraph_node (current_function_decl)); | 1879 compute_inline_parameters (cgraph_node (current_function_decl)); |
1633 return 0; | 1880 return 0; |
1634 } | 1881 } |
1635 | 1882 |
1636 struct gimple_opt_pass pass_inline_parameters = | 1883 struct gimple_opt_pass pass_inline_parameters = |
1637 { | 1884 { |
1638 { | 1885 { |
1639 GIMPLE_PASS, | 1886 GIMPLE_PASS, |
1640 NULL, /* name */ | 1887 "inline_param", /* name */ |
1641 NULL, /* gate */ | 1888 NULL, /* gate */ |
1642 compute_inline_parameters_for_current,/* execute */ | 1889 compute_inline_parameters_for_current,/* execute */ |
1643 NULL, /* sub */ | 1890 NULL, /* sub */ |
1644 NULL, /* next */ | 1891 NULL, /* next */ |
1645 0, /* static_pass_number */ | 1892 0, /* static_pass_number */ |
1646 TV_INLINE_HEURISTICS, /* tv_id */ | 1893 TV_INLINE_HEURISTICS, /* tv_id */ |
1647 0, /* properties_required */ | 1894 0, /* properties_required */ |
1648 PROP_cfg, /* properties_provided */ | 1895 0, /* properties_provided */ |
1649 0, /* properties_destroyed */ | 1896 0, /* properties_destroyed */ |
1650 0, /* todo_flags_start */ | 1897 0, /* todo_flags_start */ |
1651 0 /* todo_flags_finish */ | 1898 0 /* todo_flags_finish */ |
1652 } | 1899 } |
1653 }; | 1900 }; |
1719 } | 1966 } |
1720 | 1967 |
1721 for (node = cgraph_nodes; node; node = node->next) | 1968 for (node = cgraph_nodes; node; node = node->next) |
1722 if (node->analyzed) | 1969 if (node->analyzed) |
1723 analyze_function (node); | 1970 analyze_function (node); |
1724 | 1971 |
1725 return; | 1972 return; |
1726 } | 1973 } |
1727 | 1974 |
1728 /* Apply inline plan to function. */ | 1975 /* Apply inline plan to function. */ |
1729 static unsigned int | 1976 static unsigned int |
1730 inline_transform (struct cgraph_node *node) | 1977 inline_transform (struct cgraph_node *node) |
1731 { | 1978 { |
1732 unsigned int todo = 0; | 1979 unsigned int todo = 0; |
1733 struct cgraph_edge *e; | 1980 struct cgraph_edge *e; |
1981 | |
1982 /* FIXME: Currently the passmanager is adding inline transform more than once to some | |
1983 clones. This needs revisiting after WPA cleanups. */ | |
1984 if (cfun->after_inlining) | |
1985 return 0; | |
1734 | 1986 |
1735 /* We might need the body of this function so that we can expand | 1987 /* We might need the body of this function so that we can expand |
1736 it inline somewhere else. */ | 1988 it inline somewhere else. */ |
1737 if (cgraph_preserve_function_body_p (node->decl)) | 1989 if (cgraph_preserve_function_body_p (node->decl)) |
1738 save_inline_function_body (node); | 1990 save_inline_function_body (node); |
1745 { | 1997 { |
1746 timevar_push (TV_INTEGRATION); | 1998 timevar_push (TV_INTEGRATION); |
1747 todo = optimize_inline_calls (current_function_decl); | 1999 todo = optimize_inline_calls (current_function_decl); |
1748 timevar_pop (TV_INTEGRATION); | 2000 timevar_pop (TV_INTEGRATION); |
1749 } | 2001 } |
2002 cfun->always_inline_functions_inlined = true; | |
2003 cfun->after_inlining = true; | |
1750 return todo | execute_fixup_cfg (); | 2004 return todo | execute_fixup_cfg (); |
1751 } | 2005 } |
1752 | 2006 |
1753 struct ipa_opt_pass pass_ipa_inline = | 2007 /* Read inline summary. Jump functions are shared among ipa-cp |
2008 and inliner, so when ipa-cp is active, we don't need to write them | |
2009 twice. */ | |
2010 | |
2011 static void | |
2012 inline_read_summary (void) | |
2013 { | |
2014 if (flag_indirect_inlining) | |
2015 { | |
2016 ipa_register_cgraph_hooks (); | |
2017 if (!flag_ipa_cp) | |
2018 ipa_prop_read_jump_functions (); | |
2019 } | |
2020 function_insertion_hook_holder = | |
2021 cgraph_add_function_insertion_hook (&add_new_function, NULL); | |
2022 } | |
2023 | |
2024 /* Write inline summary for node in SET. | |
2025 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is | |
2026 active, we don't need to write them twice. */ | |
2027 | |
2028 static void | |
2029 inline_write_summary (cgraph_node_set set) | |
2030 { | |
2031 if (flag_indirect_inlining && !flag_ipa_cp) | |
2032 ipa_prop_write_jump_functions (set); | |
2033 } | |
2034 | |
2035 struct ipa_opt_pass_d pass_ipa_inline = | |
1754 { | 2036 { |
1755 { | 2037 { |
1756 IPA_PASS, | 2038 IPA_PASS, |
1757 "inline", /* name */ | 2039 "inline", /* name */ |
1758 NULL, /* gate */ | 2040 NULL, /* gate */ |
1760 NULL, /* sub */ | 2042 NULL, /* sub */ |
1761 NULL, /* next */ | 2043 NULL, /* next */ |
1762 0, /* static_pass_number */ | 2044 0, /* static_pass_number */ |
1763 TV_INLINE_HEURISTICS, /* tv_id */ | 2045 TV_INLINE_HEURISTICS, /* tv_id */ |
1764 0, /* properties_required */ | 2046 0, /* properties_required */ |
1765 PROP_cfg, /* properties_provided */ | 2047 0, /* properties_provided */ |
1766 0, /* properties_destroyed */ | 2048 0, /* properties_destroyed */ |
1767 TODO_remove_functions, /* todo_flags_finish */ | 2049 TODO_remove_functions, /* todo_flags_finish */ |
1768 TODO_dump_cgraph | TODO_dump_func | 2050 TODO_dump_cgraph | TODO_dump_func |
1769 | TODO_remove_functions /* todo_flags_finish */ | 2051 | TODO_remove_functions /* todo_flags_finish */ |
1770 }, | 2052 }, |
1771 inline_generate_summary, /* generate_summary */ | 2053 inline_generate_summary, /* generate_summary */ |
1772 NULL, /* write_summary */ | 2054 inline_write_summary, /* write_summary */ |
1773 NULL, /* read_summary */ | 2055 inline_read_summary, /* read_summary */ |
1774 NULL, /* function_read_summary */ | 2056 NULL, /* function_read_summary */ |
2057 lto_ipa_fixup_call_notes, /* stmt_fixup */ | |
1775 0, /* TODOs */ | 2058 0, /* TODOs */ |
1776 inline_transform, /* function_transform */ | 2059 inline_transform, /* function_transform */ |
1777 NULL, /* variable_transform */ | 2060 NULL, /* variable_transform */ |
1778 }; | 2061 }; |
1779 | 2062 |