|
111
|
1 /* Callgraph transformations to handle inlining
|
|
|
2 Copyright (C) 2003-2017 Free Software Foundation, Inc.
|
|
|
3 Contributed by Jan Hubicka
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
|
21 /* The inline decisions are stored in callgraph in "inline plan" and
|
|
|
22 applied later.
|
|
|
23
|
|
|
24 To mark given call inline, use inline_call function.
|
|
|
25 The function marks the edge inlinable and, if necessary, produces
|
|
|
26 virtual clone in the callgraph representing the new copy of callee's
|
|
|
27 function body.
|
|
|
28
|
|
|
29 The inline plan is applied on given function body by inline_transform. */
|
|
|
30
|
|
|
31 #include "config.h"
|
|
|
32 #include "system.h"
|
|
|
33 #include "coretypes.h"
|
|
|
34 #include "tm.h"
|
|
|
35 #include "function.h"
|
|
|
36 #include "tree.h"
|
|
|
37 #include "alloc-pool.h"
|
|
|
38 #include "tree-pass.h"
|
|
|
39 #include "cgraph.h"
|
|
|
40 #include "tree-cfg.h"
|
|
|
41 #include "symbol-summary.h"
|
|
|
42 #include "tree-vrp.h"
|
|
|
43 #include "ipa-prop.h"
|
|
|
44 #include "ipa-fnsummary.h"
|
|
|
45 #include "ipa-inline.h"
|
|
|
46 #include "tree-inline.h"
|
|
|
47 #include "function.h"
|
|
|
48 #include "cfg.h"
|
|
|
49 #include "basic-block.h"
|
|
|
50
|
|
|
51 int ncalls_inlined;
|
|
|
52 int nfunctions_inlined;
|
|
|
53
|
|
|
54 /* Scale frequency of NODE edges by FREQ_SCALE. */
|
|
|
55
|
|
|
56 static void
|
|
|
57 update_noncloned_frequencies (struct cgraph_node *node,
|
|
|
58 int freq_scale, profile_count num,
|
|
|
59 profile_count den)
|
|
|
60 {
|
|
|
61 struct cgraph_edge *e;
|
|
|
62 bool scale = (num == profile_count::zero () || den > 0);
|
|
|
63
|
|
|
64 /* We do not want to ignore high loop nest after freq drops to 0. */
|
|
|
65 if (!freq_scale)
|
|
|
66 freq_scale = 1;
|
|
|
67 for (e = node->callees; e; e = e->next_callee)
|
|
|
68 {
|
|
|
69 e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
|
|
|
70 if (e->frequency > CGRAPH_FREQ_MAX)
|
|
|
71 e->frequency = CGRAPH_FREQ_MAX;
|
|
|
72 if (!e->inline_failed)
|
|
|
73 update_noncloned_frequencies (e->callee, freq_scale, num, den);
|
|
|
74 if (scale)
|
|
|
75 e->count = e->count.apply_scale (num, den);
|
|
|
76 }
|
|
|
77 for (e = node->indirect_calls; e; e = e->next_callee)
|
|
|
78 {
|
|
|
79 e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
|
|
|
80 if (e->frequency > CGRAPH_FREQ_MAX)
|
|
|
81 e->frequency = CGRAPH_FREQ_MAX;
|
|
|
82 if (scale)
|
|
|
83 e->count = e->count.apply_scale (num, den);
|
|
|
84 }
|
|
|
85 if (scale)
|
|
|
86 node->count = node->count.apply_scale (num, den);
|
|
|
87 }
|
|
|
88
|
|
|
89 /* We removed or are going to remove the last call to NODE.
|
|
|
90 Return true if we can and want proactively remove the NODE now.
|
|
|
91 This is important to do, since we want inliner to know when offline
|
|
|
92 copy of function was removed. */
|
|
|
93
|
|
|
94 static bool
|
|
|
95 can_remove_node_now_p_1 (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
96 {
|
|
|
97 ipa_ref *ref;
|
|
|
98
|
|
|
99 FOR_EACH_ALIAS (node, ref)
|
|
|
100 {
|
|
|
101 cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
|
|
|
102 if ((alias->callers && alias->callers != e)
|
|
|
103 || !can_remove_node_now_p_1 (alias, e))
|
|
|
104 return false;
|
|
|
105 }
|
|
|
106 /* FIXME: When address is taken of DECL_EXTERNAL function we still
|
|
|
107 can remove its offline copy, but we would need to keep unanalyzed node in
|
|
|
108 the callgraph so references can point to it.
|
|
|
109
|
|
|
110 Also for comdat group we can ignore references inside a group as we
|
|
|
111 want to prove the group as a whole to be dead. */
|
|
|
112 return (!node->address_taken
|
|
|
113 && node->can_remove_if_no_direct_calls_and_refs_p ()
|
|
|
114 /* Inlining might enable more devirtualizing, so we want to remove
|
|
|
115 those only after all devirtualizable virtual calls are processed.
|
|
|
116 Lacking may edges in callgraph we just preserve them post
|
|
|
117 inlining. */
|
|
|
118 && (!DECL_VIRTUAL_P (node->decl)
|
|
|
119 || !opt_for_fn (node->decl, flag_devirtualize))
|
|
|
120 /* During early inlining some unanalyzed cgraph nodes might be in the
|
|
|
121 callgraph and they might reffer the function in question. */
|
|
|
122 && !cgraph_new_nodes.exists ());
|
|
|
123 }
|
|
|
124
|
|
|
125 /* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
|
|
|
126 Verify that the NODE can be removed from unit and if it is contained in comdat
|
|
|
127 group that the whole comdat group is removable. */
|
|
|
128
|
|
|
129 static bool
|
|
|
130 can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
131 {
|
|
|
132 struct cgraph_node *next;
|
|
|
133 if (!can_remove_node_now_p_1 (node, e))
|
|
|
134 return false;
|
|
|
135
|
|
|
136 /* When we see same comdat group, we need to be sure that all
|
|
|
137 items can be removed. */
|
|
|
138 if (!node->same_comdat_group || !node->externally_visible)
|
|
|
139 return true;
|
|
|
140 for (next = dyn_cast<cgraph_node *> (node->same_comdat_group);
|
|
|
141 next != node; next = dyn_cast<cgraph_node *> (next->same_comdat_group))
|
|
|
142 {
|
|
|
143 if (next->alias)
|
|
|
144 continue;
|
|
|
145 if ((next->callers && next->callers != e)
|
|
|
146 || !can_remove_node_now_p_1 (next, e))
|
|
|
147 return false;
|
|
|
148 }
|
|
|
149 return true;
|
|
|
150 }
|
|
|
151
|
|
|
152 /* Return true if NODE is a master clone with non-inline clones. */
|
|
|
153
|
|
|
154 static bool
|
|
|
155 master_clone_with_noninline_clones_p (struct cgraph_node *node)
|
|
|
156 {
|
|
|
157 if (node->clone_of)
|
|
|
158 return false;
|
|
|
159
|
|
|
160 for (struct cgraph_node *n = node->clones; n; n = n->next_sibling_clone)
|
|
|
161 if (n->decl != node->decl)
|
|
|
162 return true;
|
|
|
163
|
|
|
164 return false;
|
|
|
165 }
|
|
|
166
|
|
|
167 /* E is expected to be an edge being inlined. Clone destination node of
|
|
|
168 the edge and redirect it to the new clone.
|
|
|
169 DUPLICATE is used for bookkeeping on whether we are actually creating new
|
|
|
170 clones or re-using node originally representing out-of-line function call.
|
|
|
171 By default the offline copy is removed, when it appears dead after inlining.
|
|
|
172 UPDATE_ORIGINAL prevents this transformation.
|
|
|
173 If OVERALL_SIZE is non-NULL, the size is updated to reflect the
|
|
|
174 transformation.
|
|
|
175 FREQ_SCALE specify the scaling of frequencies of call sites. */
|
|
|
176
|
|
|
177 void
|
|
|
178 clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
|
|
|
179 bool update_original, int *overall_size, int freq_scale)
|
|
|
180 {
|
|
|
181 struct cgraph_node *inlining_into;
|
|
|
182 struct cgraph_edge *next;
|
|
|
183
|
|
|
184 if (e->caller->global.inlined_to)
|
|
|
185 inlining_into = e->caller->global.inlined_to;
|
|
|
186 else
|
|
|
187 inlining_into = e->caller;
|
|
|
188
|
|
|
189 if (duplicate)
|
|
|
190 {
|
|
|
191 /* We may eliminate the need for out-of-line copy to be output.
|
|
|
192 In that case just go ahead and re-use it. This is not just an
|
|
|
193 memory optimization. Making offline copy of fuction disappear
|
|
|
194 from the program will improve future decisions on inlining. */
|
|
|
195 if (!e->callee->callers->next_caller
|
|
|
196 /* Recursive inlining never wants the master clone to
|
|
|
197 be overwritten. */
|
|
|
198 && update_original
|
|
|
199 && can_remove_node_now_p (e->callee, e)
|
|
|
200 /* We cannot overwrite a master clone with non-inline clones
|
|
|
201 until after these clones are materialized. */
|
|
|
202 && !master_clone_with_noninline_clones_p (e->callee))
|
|
|
203 {
|
|
|
204 /* TODO: When callee is in a comdat group, we could remove all of it,
|
|
|
205 including all inline clones inlined into it. That would however
|
|
|
206 need small function inlining to register edge removal hook to
|
|
|
207 maintain the priority queue.
|
|
|
208
|
|
|
209 For now we keep the ohter functions in the group in program until
|
|
|
210 cgraph_remove_unreachable_functions gets rid of them. */
|
|
|
211 gcc_assert (!e->callee->global.inlined_to);
|
|
|
212 e->callee->remove_from_same_comdat_group ();
|
|
|
213 if (e->callee->definition
|
|
|
214 && inline_account_function_p (e->callee))
|
|
|
215 {
|
|
|
216 gcc_assert (!e->callee->alias);
|
|
|
217 if (overall_size)
|
|
|
218 *overall_size -= ipa_fn_summaries->get (e->callee)->size;
|
|
|
219 nfunctions_inlined++;
|
|
|
220 }
|
|
|
221 duplicate = false;
|
|
|
222 e->callee->externally_visible = false;
|
|
|
223 update_noncloned_frequencies (e->callee, e->frequency,
|
|
|
224 e->count, e->callee->count);
|
|
|
225
|
|
|
226 dump_callgraph_transformation (e->callee, inlining_into,
|
|
|
227 "inlining to");
|
|
|
228 }
|
|
|
229 else
|
|
|
230 {
|
|
|
231 struct cgraph_node *n;
|
|
|
232
|
|
|
233 if (freq_scale == -1)
|
|
|
234 freq_scale = e->frequency;
|
|
|
235 n = e->callee->create_clone (e->callee->decl,
|
|
|
236 MIN (e->count, e->callee->count),
|
|
|
237 freq_scale,
|
|
|
238 update_original, vNULL, true,
|
|
|
239 inlining_into,
|
|
|
240 NULL);
|
|
|
241 n->used_as_abstract_origin = e->callee->used_as_abstract_origin;
|
|
|
242 e->redirect_callee (n);
|
|
|
243 }
|
|
|
244 }
|
|
|
245 else
|
|
|
246 e->callee->remove_from_same_comdat_group ();
|
|
|
247
|
|
|
248 e->callee->global.inlined_to = inlining_into;
|
|
|
249
|
|
|
250 /* Recursively clone all bodies. */
|
|
|
251 for (e = e->callee->callees; e; e = next)
|
|
|
252 {
|
|
|
253 next = e->next_callee;
|
|
|
254 if (!e->inline_failed)
|
|
|
255 clone_inlined_nodes (e, duplicate, update_original, overall_size, freq_scale);
|
|
|
256 }
|
|
|
257 }
|
|
|
258
|
|
|
259 /* Check all speculations in N and resolve them if they seems useless. */
|
|
|
260
|
|
|
261 static bool
|
|
|
262 check_speculations (cgraph_node *n)
|
|
|
263 {
|
|
|
264 bool speculation_removed = false;
|
|
|
265 cgraph_edge *next;
|
|
|
266
|
|
|
267 for (cgraph_edge *e = n->callees; e; e = next)
|
|
|
268 {
|
|
|
269 next = e->next_callee;
|
|
|
270 if (e->speculative && !speculation_useful_p (e, true))
|
|
|
271 {
|
|
|
272 e->resolve_speculation (NULL);
|
|
|
273 speculation_removed = true;
|
|
|
274 }
|
|
|
275 else if (!e->inline_failed)
|
|
|
276 speculation_removed |= check_speculations (e->callee);
|
|
|
277 }
|
|
|
278 return speculation_removed;
|
|
|
279 }
|
|
|
280
|
|
|
281 /* Mark all call graph edges coming out of NODE and all nodes that have been
|
|
|
282 inlined to it as in_polymorphic_cdtor. */
|
|
|
283
|
|
|
284 static void
|
|
|
285 mark_all_inlined_calls_cdtor (cgraph_node *node)
|
|
|
286 {
|
|
|
287 for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
|
|
|
288 {
|
|
|
289 cs->in_polymorphic_cdtor = true;
|
|
|
290 if (!cs->inline_failed)
|
|
|
291 mark_all_inlined_calls_cdtor (cs->callee);
|
|
|
292 }
|
|
|
293 for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
|
|
|
294 cs->in_polymorphic_cdtor = true;
|
|
|
295 }
|
|
|
296
|
|
|
297
|
|
|
298 /* Mark edge E as inlined and update callgraph accordingly. UPDATE_ORIGINAL
|
|
|
299 specify whether profile of original function should be updated. If any new
|
|
|
300 indirect edges are discovered in the process, add them to NEW_EDGES, unless
|
|
|
301 it is NULL. If UPDATE_OVERALL_SUMMARY is false, do not bother to recompute overall
|
|
|
302 size of caller after inlining. Caller is required to eventually do it via
|
|
|
303 ipa_update_overall_fn_summary.
|
|
|
304 If callee_removed is non-NULL, set it to true if we removed callee node.
|
|
|
305
|
|
|
306 Return true iff any new callgraph edges were discovered as a
|
|
|
307 result of inlining. */
|
|
|
308
|
|
|
309 bool
|
|
|
310 inline_call (struct cgraph_edge *e, bool update_original,
|
|
|
311 vec<cgraph_edge *> *new_edges,
|
|
|
312 int *overall_size, bool update_overall_summary,
|
|
|
313 bool *callee_removed)
|
|
|
314 {
|
|
|
315 int old_size = 0, new_size = 0;
|
|
|
316 struct cgraph_node *to = NULL;
|
|
|
317 struct cgraph_edge *curr = e;
|
|
|
318 struct cgraph_node *callee = e->callee->ultimate_alias_target ();
|
|
|
319 bool new_edges_found = false;
|
|
|
320
|
|
|
321 int estimated_growth = 0;
|
|
|
322 if (! update_overall_summary)
|
|
|
323 estimated_growth = estimate_edge_growth (e);
|
|
|
324 /* This is used only for assert bellow. */
|
|
|
325 #if 0
|
|
|
326 bool predicated = inline_edge_summary (e)->predicate != NULL;
|
|
|
327 #endif
|
|
|
328
|
|
|
329 /* Don't inline inlined edges. */
|
|
|
330 gcc_assert (e->inline_failed);
|
|
|
331 /* Don't even think of inlining inline clone. */
|
|
|
332 gcc_assert (!callee->global.inlined_to);
|
|
|
333
|
|
|
334 to = e->caller;
|
|
|
335 if (to->global.inlined_to)
|
|
|
336 to = to->global.inlined_to;
|
|
|
337 if (to->thunk.thunk_p)
|
|
|
338 {
|
|
|
339 struct cgraph_node *target = to->callees->callee;
|
|
|
340 if (in_lto_p)
|
|
|
341 to->get_untransformed_body ();
|
|
|
342 to->expand_thunk (false, true);
|
|
|
343 /* When thunk is instrumented we may have multiple callees. */
|
|
|
344 for (e = to->callees; e && e->callee != target; e = e->next_callee)
|
|
|
345 ;
|
|
|
346 gcc_assert (e);
|
|
|
347 }
|
|
|
348
|
|
|
349
|
|
|
350 e->inline_failed = CIF_OK;
|
|
|
351 DECL_POSSIBLY_INLINED (callee->decl) = true;
|
|
|
352
|
|
|
353 if (DECL_FUNCTION_PERSONALITY (callee->decl))
|
|
|
354 DECL_FUNCTION_PERSONALITY (to->decl)
|
|
|
355 = DECL_FUNCTION_PERSONALITY (callee->decl);
|
|
|
356
|
|
|
357 bool reload_optimization_node = false;
|
|
|
358 if (!opt_for_fn (callee->decl, flag_strict_aliasing)
|
|
|
359 && opt_for_fn (to->decl, flag_strict_aliasing))
|
|
|
360 {
|
|
|
361 struct gcc_options opts = global_options;
|
|
|
362
|
|
|
363 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
364 opts.x_flag_strict_aliasing = false;
|
|
|
365 if (dump_file)
|
|
|
366 fprintf (dump_file, "Dropping flag_strict_aliasing on %s\n",
|
|
|
367 to->dump_name ());
|
|
|
368 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
369 = build_optimization_node (&opts);
|
|
|
370 reload_optimization_node = true;
|
|
|
371 }
|
|
|
372
|
|
|
373 ipa_fn_summary *caller_info = ipa_fn_summaries->get (to);
|
|
|
374 ipa_fn_summary *callee_info = ipa_fn_summaries->get (callee);
|
|
|
375 if (!caller_info->fp_expressions && callee_info->fp_expressions)
|
|
|
376 {
|
|
|
377 caller_info->fp_expressions = true;
|
|
|
378 if (opt_for_fn (callee->decl, flag_rounding_math)
|
|
|
379 != opt_for_fn (to->decl, flag_rounding_math)
|
|
|
380 || opt_for_fn (callee->decl, flag_trapping_math)
|
|
|
381 != opt_for_fn (to->decl, flag_trapping_math)
|
|
|
382 || opt_for_fn (callee->decl, flag_unsafe_math_optimizations)
|
|
|
383 != opt_for_fn (to->decl, flag_unsafe_math_optimizations)
|
|
|
384 || opt_for_fn (callee->decl, flag_finite_math_only)
|
|
|
385 != opt_for_fn (to->decl, flag_finite_math_only)
|
|
|
386 || opt_for_fn (callee->decl, flag_signaling_nans)
|
|
|
387 != opt_for_fn (to->decl, flag_signaling_nans)
|
|
|
388 || opt_for_fn (callee->decl, flag_cx_limited_range)
|
|
|
389 != opt_for_fn (to->decl, flag_cx_limited_range)
|
|
|
390 || opt_for_fn (callee->decl, flag_signed_zeros)
|
|
|
391 != opt_for_fn (to->decl, flag_signed_zeros)
|
|
|
392 || opt_for_fn (callee->decl, flag_associative_math)
|
|
|
393 != opt_for_fn (to->decl, flag_associative_math)
|
|
|
394 || opt_for_fn (callee->decl, flag_reciprocal_math)
|
|
|
395 != opt_for_fn (to->decl, flag_reciprocal_math)
|
|
|
396 || opt_for_fn (callee->decl, flag_fp_int_builtin_inexact)
|
|
|
397 != opt_for_fn (to->decl, flag_fp_int_builtin_inexact)
|
|
|
398 || opt_for_fn (callee->decl, flag_errno_math)
|
|
|
399 != opt_for_fn (to->decl, flag_errno_math))
|
|
|
400 {
|
|
|
401 struct gcc_options opts = global_options;
|
|
|
402
|
|
|
403 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
404 opts.x_flag_rounding_math
|
|
|
405 = opt_for_fn (callee->decl, flag_rounding_math);
|
|
|
406 opts.x_flag_trapping_math
|
|
|
407 = opt_for_fn (callee->decl, flag_trapping_math);
|
|
|
408 opts.x_flag_unsafe_math_optimizations
|
|
|
409 = opt_for_fn (callee->decl, flag_unsafe_math_optimizations);
|
|
|
410 opts.x_flag_finite_math_only
|
|
|
411 = opt_for_fn (callee->decl, flag_finite_math_only);
|
|
|
412 opts.x_flag_signaling_nans
|
|
|
413 = opt_for_fn (callee->decl, flag_signaling_nans);
|
|
|
414 opts.x_flag_cx_limited_range
|
|
|
415 = opt_for_fn (callee->decl, flag_cx_limited_range);
|
|
|
416 opts.x_flag_signed_zeros
|
|
|
417 = opt_for_fn (callee->decl, flag_signed_zeros);
|
|
|
418 opts.x_flag_associative_math
|
|
|
419 = opt_for_fn (callee->decl, flag_associative_math);
|
|
|
420 opts.x_flag_reciprocal_math
|
|
|
421 = opt_for_fn (callee->decl, flag_reciprocal_math);
|
|
|
422 opts.x_flag_fp_int_builtin_inexact
|
|
|
423 = opt_for_fn (callee->decl, flag_fp_int_builtin_inexact);
|
|
|
424 opts.x_flag_errno_math
|
|
|
425 = opt_for_fn (callee->decl, flag_errno_math);
|
|
|
426 if (dump_file)
|
|
|
427 fprintf (dump_file, "Copying FP flags from %s to %s\n",
|
|
|
428 callee->dump_name (), to->dump_name ());
|
|
|
429 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
430 = build_optimization_node (&opts);
|
|
|
431 reload_optimization_node = true;
|
|
|
432 }
|
|
|
433 }
|
|
|
434
|
|
|
435 /* Reload global optimization flags. */
|
|
|
436 if (reload_optimization_node && DECL_STRUCT_FUNCTION (to->decl) == cfun)
|
|
|
437 set_cfun (cfun, true);
|
|
|
438
|
|
|
439 /* If aliases are involved, redirect edge to the actual destination and
|
|
|
440 possibly remove the aliases. */
|
|
|
441 if (e->callee != callee)
|
|
|
442 {
|
|
|
443 struct cgraph_node *alias = e->callee, *next_alias;
|
|
|
444 e->redirect_callee (callee);
|
|
|
445 while (alias && alias != callee)
|
|
|
446 {
|
|
|
447 if (!alias->callers
|
|
|
448 && can_remove_node_now_p (alias,
|
|
|
449 !e->next_caller && !e->prev_caller ? e : NULL))
|
|
|
450 {
|
|
|
451 next_alias = alias->get_alias_target ();
|
|
|
452 alias->remove ();
|
|
|
453 if (callee_removed)
|
|
|
454 *callee_removed = true;
|
|
|
455 alias = next_alias;
|
|
|
456 }
|
|
|
457 else
|
|
|
458 break;
|
|
|
459 }
|
|
|
460 }
|
|
|
461
|
|
|
462 clone_inlined_nodes (e, true, update_original, overall_size, e->frequency);
|
|
|
463
|
|
|
464 gcc_assert (curr->callee->global.inlined_to == to);
|
|
|
465
|
|
|
466 old_size = ipa_fn_summaries->get (to)->size;
|
|
|
467 ipa_merge_fn_summary_after_inlining (e);
|
|
|
468 if (e->in_polymorphic_cdtor)
|
|
|
469 mark_all_inlined_calls_cdtor (e->callee);
|
|
|
470 if (opt_for_fn (e->caller->decl, optimize))
|
|
|
471 new_edges_found = ipa_propagate_indirect_call_infos (curr, new_edges);
|
|
|
472 check_speculations (e->callee);
|
|
|
473 if (update_overall_summary)
|
|
|
474 ipa_update_overall_fn_summary (to);
|
|
|
475 else
|
|
|
476 /* Update self size by the estimate so overall function growth limits
|
|
|
477 work for further inlining into this function. Before inlining
|
|
|
478 the function we inlined to again we expect the caller to update
|
|
|
479 the overall summary. */
|
|
|
480 ipa_fn_summaries->get (to)->size += estimated_growth;
|
|
|
481 new_size = ipa_fn_summaries->get (to)->size;
|
|
|
482
|
|
|
483 if (callee->calls_comdat_local)
|
|
|
484 to->calls_comdat_local = true;
|
|
|
485 else if (to->calls_comdat_local && callee->comdat_local_p ())
|
|
|
486 {
|
|
|
487 struct cgraph_edge *se = to->callees;
|
|
|
488 for (; se; se = se->next_callee)
|
|
|
489 if (se->inline_failed && se->callee->comdat_local_p ())
|
|
|
490 break;
|
|
|
491 if (se == NULL)
|
|
|
492 to->calls_comdat_local = false;
|
|
|
493 }
|
|
|
494
|
|
|
495 /* FIXME: This assert suffers from roundoff errors, disable it for GCC 5
|
|
|
496 and revisit it after conversion to sreals in GCC 6.
|
|
|
497 See PR 65654. */
|
|
|
498 #if 0
|
|
|
499 /* Verify that estimated growth match real growth. Allow off-by-one
|
|
|
500 error due to ipa_fn_summary::size_scale roudoff errors. */
|
|
|
501 gcc_assert (!update_overall_summary || !overall_size || new_edges_found
|
|
|
502 || abs (estimated_growth - (new_size - old_size)) <= 1
|
|
|
503 || speculation_removed
|
|
|
504 /* FIXME: a hack. Edges with false predicate are accounted
|
|
|
505 wrong, we should remove them from callgraph. */
|
|
|
506 || predicated);
|
|
|
507 #endif
|
|
|
508
|
|
|
509 /* Account the change of overall unit size; external functions will be
|
|
|
510 removed and are thus not accounted. */
|
|
|
511 if (overall_size && inline_account_function_p (to))
|
|
|
512 *overall_size += new_size - old_size;
|
|
|
513 ncalls_inlined++;
|
|
|
514
|
|
|
515 /* This must happen after ipa_merge_fn_summary_after_inlining that rely on jump
|
|
|
516 functions of callee to not be updated. */
|
|
|
517 return new_edges_found;
|
|
|
518 }
|
|
|
519
|
|
|
520
|
|
|
521 /* Copy function body of NODE and redirect all inline clones to it.
|
|
|
522 This is done before inline plan is applied to NODE when there are
|
|
|
523 still some inline clones if it.
|
|
|
524
|
|
|
525 This is necessary because inline decisions are not really transitive
|
|
|
526 and the other inline clones may have different bodies. */
|
|
|
527
|
|
|
528 static struct cgraph_node *
|
|
|
529 save_inline_function_body (struct cgraph_node *node)
|
|
|
530 {
|
|
|
531 struct cgraph_node *first_clone, *n;
|
|
|
532
|
|
|
533 if (dump_file)
|
|
|
534 fprintf (dump_file, "\nSaving body of %s for later reuse\n",
|
|
|
535 node->name ());
|
|
|
536
|
|
|
537 gcc_assert (node == cgraph_node::get (node->decl));
|
|
|
538
|
|
|
539 /* first_clone will be turned into real function. */
|
|
|
540 first_clone = node->clones;
|
|
|
541
|
|
|
542 /* Arrange first clone to not be thunk as those do not have bodies. */
|
|
|
543 if (first_clone->thunk.thunk_p)
|
|
|
544 {
|
|
|
545 while (first_clone->thunk.thunk_p)
|
|
|
546 first_clone = first_clone->next_sibling_clone;
|
|
|
547 first_clone->prev_sibling_clone->next_sibling_clone
|
|
|
548 = first_clone->next_sibling_clone;
|
|
|
549 if (first_clone->next_sibling_clone)
|
|
|
550 first_clone->next_sibling_clone->prev_sibling_clone
|
|
|
551 = first_clone->prev_sibling_clone;
|
|
|
552 first_clone->next_sibling_clone = node->clones;
|
|
|
553 first_clone->prev_sibling_clone = NULL;
|
|
|
554 node->clones->prev_sibling_clone = first_clone;
|
|
|
555 node->clones = first_clone;
|
|
|
556 }
|
|
|
557 first_clone->decl = copy_node (node->decl);
|
|
|
558 first_clone->decl->decl_with_vis.symtab_node = first_clone;
|
|
|
559 gcc_assert (first_clone == cgraph_node::get (first_clone->decl));
|
|
|
560
|
|
|
561 /* Now reshape the clone tree, so all other clones descends from
|
|
|
562 first_clone. */
|
|
|
563 if (first_clone->next_sibling_clone)
|
|
|
564 {
|
|
|
565 for (n = first_clone->next_sibling_clone; n->next_sibling_clone;
|
|
|
566 n = n->next_sibling_clone)
|
|
|
567 n->clone_of = first_clone;
|
|
|
568 n->clone_of = first_clone;
|
|
|
569 n->next_sibling_clone = first_clone->clones;
|
|
|
570 if (first_clone->clones)
|
|
|
571 first_clone->clones->prev_sibling_clone = n;
|
|
|
572 first_clone->clones = first_clone->next_sibling_clone;
|
|
|
573 first_clone->next_sibling_clone->prev_sibling_clone = NULL;
|
|
|
574 first_clone->next_sibling_clone = NULL;
|
|
|
575 gcc_assert (!first_clone->prev_sibling_clone);
|
|
|
576 }
|
|
|
577 first_clone->clone_of = NULL;
|
|
|
578
|
|
|
579 /* Now node in question has no clones. */
|
|
|
580 node->clones = NULL;
|
|
|
581
|
|
|
582 /* Inline clones share decl with the function they are cloned
|
|
|
583 from. Walk the whole clone tree and redirect them all to the
|
|
|
584 new decl. */
|
|
|
585 if (first_clone->clones)
|
|
|
586 for (n = first_clone->clones; n != first_clone;)
|
|
|
587 {
|
|
|
588 gcc_assert (n->decl == node->decl);
|
|
|
589 n->decl = first_clone->decl;
|
|
|
590 if (n->clones)
|
|
|
591 n = n->clones;
|
|
|
592 else if (n->next_sibling_clone)
|
|
|
593 n = n->next_sibling_clone;
|
|
|
594 else
|
|
|
595 {
|
|
|
596 while (n != first_clone && !n->next_sibling_clone)
|
|
|
597 n = n->clone_of;
|
|
|
598 if (n != first_clone)
|
|
|
599 n = n->next_sibling_clone;
|
|
|
600 }
|
|
|
601 }
|
|
|
602
|
|
|
603 /* Copy the OLD_VERSION_NODE function tree to the new version. */
|
|
|
604 tree_function_versioning (node->decl, first_clone->decl,
|
|
|
605 NULL, true, NULL, false,
|
|
|
606 NULL, NULL);
|
|
|
607
|
|
|
608 /* The function will be short lived and removed after we inline all the clones,
|
|
|
609 but make it internal so we won't confuse ourself. */
|
|
|
610 DECL_EXTERNAL (first_clone->decl) = 0;
|
|
|
611 TREE_PUBLIC (first_clone->decl) = 0;
|
|
|
612 DECL_COMDAT (first_clone->decl) = 0;
|
|
|
613 first_clone->ipa_transforms_to_apply.release ();
|
|
|
614
|
|
|
615 /* When doing recursive inlining, the clone may become unnecessary.
|
|
|
616 This is possible i.e. in the case when the recursive function is proved to be
|
|
|
617 non-throwing and the recursion happens only in the EH landing pad.
|
|
|
618 We can not remove the clone until we are done with saving the body.
|
|
|
619 Remove it now. */
|
|
|
620 if (!first_clone->callers)
|
|
|
621 {
|
|
|
622 first_clone->remove_symbol_and_inline_clones ();
|
|
|
623 first_clone = NULL;
|
|
|
624 }
|
|
|
625 else if (flag_checking)
|
|
|
626 first_clone->verify ();
|
|
|
627
|
|
|
628 return first_clone;
|
|
|
629 }
|
|
|
630
|
|
|
631 /* Return true when function body of DECL still needs to be kept around
|
|
|
632 for later re-use. */
|
|
|
633 static bool
|
|
|
634 preserve_function_body_p (struct cgraph_node *node)
|
|
|
635 {
|
|
|
636 gcc_assert (symtab->global_info_ready);
|
|
|
637 gcc_assert (!node->alias && !node->thunk.thunk_p);
|
|
|
638
|
|
|
639 /* Look if there is any non-thunk clone around. */
|
|
|
640 for (node = node->clones; node; node = node->next_sibling_clone)
|
|
|
641 if (!node->thunk.thunk_p)
|
|
|
642 return true;
|
|
|
643 return false;
|
|
|
644 }
|
|
|
645
|
|
|
646 /* Apply inline plan to function. */
|
|
|
647
|
|
|
648 unsigned int
|
|
|
649 inline_transform (struct cgraph_node *node)
|
|
|
650 {
|
|
|
651 unsigned int todo = 0;
|
|
|
652 struct cgraph_edge *e, *next;
|
|
|
653 bool has_inline = false;
|
|
|
654
|
|
|
655 /* FIXME: Currently the pass manager is adding inline transform more than
|
|
|
656 once to some clones. This needs revisiting after WPA cleanups. */
|
|
|
657 if (cfun->after_inlining)
|
|
|
658 return 0;
|
|
|
659
|
|
|
660 /* We might need the body of this function so that we can expand
|
|
|
661 it inline somewhere else. */
|
|
|
662 if (preserve_function_body_p (node))
|
|
|
663 save_inline_function_body (node);
|
|
|
664
|
|
|
665 for (e = node->callees; e; e = next)
|
|
|
666 {
|
|
|
667 if (!e->inline_failed)
|
|
|
668 has_inline = true;
|
|
|
669 next = e->next_callee;
|
|
|
670 e->redirect_call_stmt_to_callee ();
|
|
|
671 }
|
|
|
672 node->remove_all_references ();
|
|
|
673
|
|
|
674 timevar_push (TV_INTEGRATION);
|
|
|
675 if (node->callees && (opt_for_fn (node->decl, optimize) || has_inline))
|
|
|
676 {
|
|
|
677 profile_count num = node->count;
|
|
|
678 profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
|
679 bool scale = num.initialized_p ()
|
|
|
680 && (den > 0 || num == profile_count::zero ())
|
|
|
681 && !(num == den);
|
|
|
682 if (scale)
|
|
|
683 {
|
|
|
684 if (dump_file)
|
|
|
685 {
|
|
|
686 fprintf (dump_file, "Applying count scale ");
|
|
|
687 num.dump (dump_file);
|
|
|
688 fprintf (dump_file, "/");
|
|
|
689 den.dump (dump_file);
|
|
|
690 fprintf (dump_file, "\n");
|
|
|
691 }
|
|
|
692
|
|
|
693 basic_block bb;
|
|
|
694 FOR_ALL_BB_FN (bb, cfun)
|
|
|
695 bb->count = bb->count.apply_scale (num, den);
|
|
|
696 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
|
|
|
697 }
|
|
|
698 todo = optimize_inline_calls (current_function_decl);
|
|
|
699 }
|
|
|
700 timevar_pop (TV_INTEGRATION);
|
|
|
701
|
|
|
702 cfun->always_inline_functions_inlined = true;
|
|
|
703 cfun->after_inlining = true;
|
|
|
704 todo |= execute_fixup_cfg ();
|
|
|
705
|
|
|
706 if (!(todo & TODO_update_ssa_any))
|
|
|
707 /* Redirecting edges might lead to a need for vops to be recomputed. */
|
|
|
708 todo |= TODO_update_ssa_only_virtuals;
|
|
|
709
|
|
|
710 return todo;
|
|
|
711 }
|
