|
111
|
1 /* Callgraph transformations to handle inlining
|
|
145
|
2 Copyright (C) 2003-2020 Free Software Foundation, Inc.
|
|
111
|
3 Contributed by Jan Hubicka
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
|
21 /* The inline decisions are stored in callgraph in "inline plan" and
|
|
|
22 applied later.
|
|
|
23
|
|
|
24 To mark given call inline, use inline_call function.
|
|
|
25 The function marks the edge inlinable and, if necessary, produces
|
|
|
26 virtual clone in the callgraph representing the new copy of callee's
|
|
|
27 function body.
|
|
|
28
|
|
|
29 The inline plan is applied on given function body by inline_transform. */
|
|
|
30
|
|
|
31 #include "config.h"
|
|
|
32 #include "system.h"
|
|
|
33 #include "coretypes.h"
|
|
|
34 #include "tm.h"
|
|
|
35 #include "function.h"
|
|
|
36 #include "tree.h"
|
|
|
37 #include "alloc-pool.h"
|
|
|
38 #include "tree-pass.h"
|
|
|
39 #include "cgraph.h"
|
|
|
40 #include "tree-cfg.h"
|
|
|
41 #include "symbol-summary.h"
|
|
|
42 #include "tree-vrp.h"
|
|
|
43 #include "ipa-prop.h"
|
|
|
44 #include "ipa-fnsummary.h"
|
|
|
45 #include "ipa-inline.h"
|
|
|
46 #include "tree-inline.h"
|
|
|
47 #include "function.h"
|
|
|
48 #include "cfg.h"
|
|
|
49 #include "basic-block.h"
|
|
145
|
50 #include "ipa-utils.h"
|
|
111
|
51
|
|
|
52 int ncalls_inlined;
|
|
|
53 int nfunctions_inlined;
|
|
|
54
|
|
131
|
55 /* Scale counts of NODE edges by NUM/DEN. */
|
|
111
|
56
|
|
|
57 static void
|
|
131
|
58 update_noncloned_counts (struct cgraph_node *node,
|
|
|
59 profile_count num, profile_count den)
|
|
111
|
60 {
|
|
|
61 struct cgraph_edge *e;
|
|
|
62
|
|
131
|
63 profile_count::adjust_for_ipa_scaling (&num, &den);
|
|
|
64
|
|
111
|
65 for (e = node->callees; e; e = e->next_callee)
|
|
|
66 {
|
|
|
67 if (!e->inline_failed)
|
|
131
|
68 update_noncloned_counts (e->callee, num, den);
|
|
|
69 e->count = e->count.apply_scale (num, den);
|
|
111
|
70 }
|
|
|
71 for (e = node->indirect_calls; e; e = e->next_callee)
|
|
131
|
72 e->count = e->count.apply_scale (num, den);
|
|
|
73 node->count = node->count.apply_scale (num, den);
|
|
111
|
74 }
|
|
|
75
|
|
|
76 /* We removed or are going to remove the last call to NODE.
|
|
|
77 Return true if we can and want proactively remove the NODE now.
|
|
|
78 This is important to do, since we want inliner to know when offline
|
|
|
79 copy of function was removed. */
|
|
|
80
|
|
|
81 static bool
|
|
|
82 can_remove_node_now_p_1 (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
83 {
|
|
|
84 ipa_ref *ref;
|
|
|
85
|
|
|
86 FOR_EACH_ALIAS (node, ref)
|
|
|
87 {
|
|
|
88 cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
|
|
|
89 if ((alias->callers && alias->callers != e)
|
|
|
90 || !can_remove_node_now_p_1 (alias, e))
|
|
|
91 return false;
|
|
|
92 }
|
|
|
93 /* FIXME: When address is taken of DECL_EXTERNAL function we still
|
|
|
94 can remove its offline copy, but we would need to keep unanalyzed node in
|
|
|
95 the callgraph so references can point to it.
|
|
|
96
|
|
|
97 Also for comdat group we can ignore references inside a group as we
|
|
|
98 want to prove the group as a whole to be dead. */
|
|
|
99 return (!node->address_taken
|
|
|
100 && node->can_remove_if_no_direct_calls_and_refs_p ()
|
|
|
101 /* Inlining might enable more devirtualizing, so we want to remove
|
|
|
102 those only after all devirtualizable virtual calls are processed.
|
|
|
103 Lacking may edges in callgraph we just preserve them post
|
|
|
104 inlining. */
|
|
|
105 && (!DECL_VIRTUAL_P (node->decl)
|
|
|
106 || !opt_for_fn (node->decl, flag_devirtualize))
|
|
|
107 /* During early inlining some unanalyzed cgraph nodes might be in the
|
|
145
|
108 callgraph and they might refer the function in question. */
|
|
111
|
109 && !cgraph_new_nodes.exists ());
|
|
|
110 }
|
|
|
111
|
|
|
112 /* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
|
|
|
113 Verify that the NODE can be removed from unit and if it is contained in comdat
|
|
|
114 group that the whole comdat group is removable. */
|
|
|
115
|
|
|
116 static bool
|
|
|
117 can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
118 {
|
|
|
119 struct cgraph_node *next;
|
|
|
120 if (!can_remove_node_now_p_1 (node, e))
|
|
|
121 return false;
|
|
|
122
|
|
|
123 /* When we see same comdat group, we need to be sure that all
|
|
|
124 items can be removed. */
|
|
|
125 if (!node->same_comdat_group || !node->externally_visible)
|
|
|
126 return true;
|
|
|
127 for (next = dyn_cast<cgraph_node *> (node->same_comdat_group);
|
|
|
128 next != node; next = dyn_cast<cgraph_node *> (next->same_comdat_group))
|
|
|
129 {
|
|
|
130 if (next->alias)
|
|
|
131 continue;
|
|
|
132 if ((next->callers && next->callers != e)
|
|
|
133 || !can_remove_node_now_p_1 (next, e))
|
|
|
134 return false;
|
|
|
135 }
|
|
|
136 return true;
|
|
|
137 }
|
|
|
138
|
|
|
139 /* Return true if NODE is a master clone with non-inline clones. */
|
|
|
140
|
|
|
141 static bool
|
|
|
142 master_clone_with_noninline_clones_p (struct cgraph_node *node)
|
|
|
143 {
|
|
|
144 if (node->clone_of)
|
|
|
145 return false;
|
|
|
146
|
|
|
147 for (struct cgraph_node *n = node->clones; n; n = n->next_sibling_clone)
|
|
|
148 if (n->decl != node->decl)
|
|
|
149 return true;
|
|
|
150
|
|
|
151 return false;
|
|
|
152 }
|
|
|
153
|
|
|
154 /* E is expected to be an edge being inlined. Clone destination node of
|
|
|
155 the edge and redirect it to the new clone.
|
|
|
156 DUPLICATE is used for bookkeeping on whether we are actually creating new
|
|
|
157 clones or re-using node originally representing out-of-line function call.
|
|
|
158 By default the offline copy is removed, when it appears dead after inlining.
|
|
|
159 UPDATE_ORIGINAL prevents this transformation.
|
|
|
160 If OVERALL_SIZE is non-NULL, the size is updated to reflect the
|
|
131
|
161 transformation. */
|
|
111
|
162
|
|
|
163 void
|
|
|
164 clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
|
|
131
|
165 bool update_original, int *overall_size)
|
|
111
|
166 {
|
|
|
167 struct cgraph_node *inlining_into;
|
|
|
168 struct cgraph_edge *next;
|
|
|
169
|
|
145
|
170 if (e->caller->inlined_to)
|
|
|
171 inlining_into = e->caller->inlined_to;
|
|
111
|
172 else
|
|
|
173 inlining_into = e->caller;
|
|
|
174
|
|
|
175 if (duplicate)
|
|
|
176 {
|
|
|
177 /* We may eliminate the need for out-of-line copy to be output.
|
|
|
178 In that case just go ahead and re-use it. This is not just an
|
|
145
|
179 memory optimization. Making offline copy of function disappear
|
|
111
|
180 from the program will improve future decisions on inlining. */
|
|
|
181 if (!e->callee->callers->next_caller
|
|
|
182 /* Recursive inlining never wants the master clone to
|
|
|
183 be overwritten. */
|
|
|
184 && update_original
|
|
|
185 && can_remove_node_now_p (e->callee, e)
|
|
|
186 /* We cannot overwrite a master clone with non-inline clones
|
|
|
187 until after these clones are materialized. */
|
|
|
188 && !master_clone_with_noninline_clones_p (e->callee))
|
|
|
189 {
|
|
|
190 /* TODO: When callee is in a comdat group, we could remove all of it,
|
|
|
191 including all inline clones inlined into it. That would however
|
|
|
192 need small function inlining to register edge removal hook to
|
|
|
193 maintain the priority queue.
|
|
|
194
|
|
145
|
195 For now we keep the other functions in the group in program until
|
|
111
|
196 cgraph_remove_unreachable_functions gets rid of them. */
|
|
145
|
197 gcc_assert (!e->callee->inlined_to);
|
|
111
|
198 e->callee->remove_from_same_comdat_group ();
|
|
|
199 if (e->callee->definition
|
|
|
200 && inline_account_function_p (e->callee))
|
|
|
201 {
|
|
|
202 gcc_assert (!e->callee->alias);
|
|
|
203 if (overall_size)
|
|
145
|
204 *overall_size -= ipa_size_summaries->get (e->callee)->size;
|
|
111
|
205 nfunctions_inlined++;
|
|
|
206 }
|
|
|
207 duplicate = false;
|
|
|
208 e->callee->externally_visible = false;
|
|
131
|
209 update_noncloned_counts (e->callee, e->count, e->callee->count);
|
|
111
|
210
|
|
|
211 dump_callgraph_transformation (e->callee, inlining_into,
|
|
|
212 "inlining to");
|
|
|
213 }
|
|
|
214 else
|
|
|
215 {
|
|
|
216 struct cgraph_node *n;
|
|
|
217
|
|
|
218 n = e->callee->create_clone (e->callee->decl,
|
|
131
|
219 e->count,
|
|
111
|
220 update_original, vNULL, true,
|
|
|
221 inlining_into,
|
|
|
222 NULL);
|
|
|
223 n->used_as_abstract_origin = e->callee->used_as_abstract_origin;
|
|
|
224 e->redirect_callee (n);
|
|
|
225 }
|
|
|
226 }
|
|
|
227 else
|
|
|
228 e->callee->remove_from_same_comdat_group ();
|
|
|
229
|
|
145
|
230 e->callee->inlined_to = inlining_into;
|
|
111
|
231
|
|
|
232 /* Recursively clone all bodies. */
|
|
|
233 for (e = e->callee->callees; e; e = next)
|
|
|
234 {
|
|
|
235 next = e->next_callee;
|
|
|
236 if (!e->inline_failed)
|
|
131
|
237 clone_inlined_nodes (e, duplicate, update_original, overall_size);
|
|
111
|
238 }
|
|
|
239 }
|
|
|
240
|
|
145
|
241 /* Check all speculations in N and if any seem useless, resolve them. When a
|
|
|
242 first edge is resolved, pop all edges from NEW_EDGES and insert them to
|
|
|
243 EDGE_SET. Then remove each resolved edge from EDGE_SET, if it is there. */
|
|
111
|
244
|
|
|
245 static bool
|
|
145
|
246 check_speculations_1 (cgraph_node *n, vec<cgraph_edge *> *new_edges,
|
|
|
247 hash_set <cgraph_edge *> *edge_set)
|
|
111
|
248 {
|
|
|
249 bool speculation_removed = false;
|
|
|
250 cgraph_edge *next;
|
|
|
251
|
|
|
252 for (cgraph_edge *e = n->callees; e; e = next)
|
|
|
253 {
|
|
|
254 next = e->next_callee;
|
|
|
255 if (e->speculative && !speculation_useful_p (e, true))
|
|
|
256 {
|
|
145
|
257 while (new_edges && !new_edges->is_empty ())
|
|
|
258 edge_set->add (new_edges->pop ());
|
|
|
259 edge_set->remove (e);
|
|
|
260
|
|
|
261 cgraph_edge::resolve_speculation (e, NULL);
|
|
111
|
262 speculation_removed = true;
|
|
|
263 }
|
|
|
264 else if (!e->inline_failed)
|
|
145
|
265 speculation_removed |= check_speculations_1 (e->callee, new_edges,
|
|
|
266 edge_set);
|
|
111
|
267 }
|
|
|
268 return speculation_removed;
|
|
|
269 }
|
|
|
270
|
|
145
|
271 /* Push E to NEW_EDGES. Called from hash_set traverse method, which
|
|
|
272 unfortunately means this function has to have external linkage, otherwise
|
|
|
273 the code will not compile with gcc 4.8. */
|
|
|
274
|
|
|
275 bool
|
|
|
276 push_all_edges_in_set_to_vec (cgraph_edge * const &e,
|
|
|
277 vec<cgraph_edge *> *new_edges)
|
|
|
278 {
|
|
|
279 new_edges->safe_push (e);
|
|
|
280 return true;
|
|
|
281 }
|
|
|
282
|
|
|
283 /* Check all speculations in N and if any seem useless, resolve them and remove
|
|
|
284 them from NEW_EDGES. */
|
|
|
285
|
|
|
286 static bool
|
|
|
287 check_speculations (cgraph_node *n, vec<cgraph_edge *> *new_edges)
|
|
|
288 {
|
|
|
289 hash_set <cgraph_edge *> edge_set;
|
|
|
290 bool res = check_speculations_1 (n, new_edges, &edge_set);
|
|
|
291 if (!edge_set.is_empty ())
|
|
|
292 edge_set.traverse <vec<cgraph_edge *> *,
|
|
|
293 push_all_edges_in_set_to_vec> (new_edges);
|
|
|
294 return res;
|
|
|
295 }
|
|
|
296
|
|
111
|
297 /* Mark all call graph edges coming out of NODE and all nodes that have been
|
|
|
298 inlined to it as in_polymorphic_cdtor. */
|
|
|
299
|
|
|
300 static void
|
|
|
301 mark_all_inlined_calls_cdtor (cgraph_node *node)
|
|
|
302 {
|
|
|
303 for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
|
|
|
304 {
|
|
|
305 cs->in_polymorphic_cdtor = true;
|
|
|
306 if (!cs->inline_failed)
|
|
|
307 mark_all_inlined_calls_cdtor (cs->callee);
|
|
|
308 }
|
|
|
309 for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
|
|
|
310 cs->in_polymorphic_cdtor = true;
|
|
|
311 }
|
|
|
312
|
|
|
313
|
|
|
314 /* Mark edge E as inlined and update callgraph accordingly. UPDATE_ORIGINAL
|
|
|
315 specify whether profile of original function should be updated. If any new
|
|
|
316 indirect edges are discovered in the process, add them to NEW_EDGES, unless
|
|
|
317 it is NULL. If UPDATE_OVERALL_SUMMARY is false, do not bother to recompute overall
|
|
|
318 size of caller after inlining. Caller is required to eventually do it via
|
|
|
319 ipa_update_overall_fn_summary.
|
|
|
320 If callee_removed is non-NULL, set it to true if we removed callee node.
|
|
|
321
|
|
|
322 Return true iff any new callgraph edges were discovered as a
|
|
|
323 result of inlining. */
|
|
|
324
|
|
|
325 bool
|
|
|
326 inline_call (struct cgraph_edge *e, bool update_original,
|
|
|
327 vec<cgraph_edge *> *new_edges,
|
|
|
328 int *overall_size, bool update_overall_summary,
|
|
|
329 bool *callee_removed)
|
|
|
330 {
|
|
|
331 int old_size = 0, new_size = 0;
|
|
|
332 struct cgraph_node *to = NULL;
|
|
|
333 struct cgraph_edge *curr = e;
|
|
145
|
334 bool comdat_local = e->callee->comdat_local_p ();
|
|
111
|
335 struct cgraph_node *callee = e->callee->ultimate_alias_target ();
|
|
|
336 bool new_edges_found = false;
|
|
|
337
|
|
|
338 int estimated_growth = 0;
|
|
|
339 if (! update_overall_summary)
|
|
|
340 estimated_growth = estimate_edge_growth (e);
|
|
|
341 /* This is used only for assert bellow. */
|
|
|
342 #if 0
|
|
|
343 bool predicated = inline_edge_summary (e)->predicate != NULL;
|
|
|
344 #endif
|
|
|
345
|
|
|
346 /* Don't inline inlined edges. */
|
|
|
347 gcc_assert (e->inline_failed);
|
|
|
348 /* Don't even think of inlining inline clone. */
|
|
145
|
349 gcc_assert (!callee->inlined_to);
|
|
111
|
350
|
|
|
351 to = e->caller;
|
|
145
|
352 if (to->inlined_to)
|
|
|
353 to = to->inlined_to;
|
|
111
|
354 if (to->thunk.thunk_p)
|
|
|
355 {
|
|
|
356 struct cgraph_node *target = to->callees->callee;
|
|
145
|
357 thunk_expansion = true;
|
|
|
358 symtab->call_cgraph_removal_hooks (to);
|
|
111
|
359 if (in_lto_p)
|
|
|
360 to->get_untransformed_body ();
|
|
|
361 to->expand_thunk (false, true);
|
|
|
362 /* When thunk is instrumented we may have multiple callees. */
|
|
|
363 for (e = to->callees; e && e->callee != target; e = e->next_callee)
|
|
|
364 ;
|
|
145
|
365 symtab->call_cgraph_insertion_hooks (to);
|
|
|
366 thunk_expansion = false;
|
|
111
|
367 gcc_assert (e);
|
|
|
368 }
|
|
|
369
|
|
|
370
|
|
|
371 e->inline_failed = CIF_OK;
|
|
|
372 DECL_POSSIBLY_INLINED (callee->decl) = true;
|
|
|
373
|
|
|
374 if (DECL_FUNCTION_PERSONALITY (callee->decl))
|
|
|
375 DECL_FUNCTION_PERSONALITY (to->decl)
|
|
|
376 = DECL_FUNCTION_PERSONALITY (callee->decl);
|
|
|
377
|
|
|
378 bool reload_optimization_node = false;
|
|
|
379 if (!opt_for_fn (callee->decl, flag_strict_aliasing)
|
|
|
380 && opt_for_fn (to->decl, flag_strict_aliasing))
|
|
|
381 {
|
|
|
382 struct gcc_options opts = global_options;
|
|
|
383
|
|
|
384 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
385 opts.x_flag_strict_aliasing = false;
|
|
|
386 if (dump_file)
|
|
|
387 fprintf (dump_file, "Dropping flag_strict_aliasing on %s\n",
|
|
|
388 to->dump_name ());
|
|
|
389 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
390 = build_optimization_node (&opts);
|
|
|
391 reload_optimization_node = true;
|
|
|
392 }
|
|
|
393
|
|
|
394 ipa_fn_summary *caller_info = ipa_fn_summaries->get (to);
|
|
|
395 ipa_fn_summary *callee_info = ipa_fn_summaries->get (callee);
|
|
|
396 if (!caller_info->fp_expressions && callee_info->fp_expressions)
|
|
|
397 {
|
|
|
398 caller_info->fp_expressions = true;
|
|
|
399 if (opt_for_fn (callee->decl, flag_rounding_math)
|
|
|
400 != opt_for_fn (to->decl, flag_rounding_math)
|
|
|
401 || opt_for_fn (callee->decl, flag_trapping_math)
|
|
|
402 != opt_for_fn (to->decl, flag_trapping_math)
|
|
|
403 || opt_for_fn (callee->decl, flag_unsafe_math_optimizations)
|
|
|
404 != opt_for_fn (to->decl, flag_unsafe_math_optimizations)
|
|
|
405 || opt_for_fn (callee->decl, flag_finite_math_only)
|
|
|
406 != opt_for_fn (to->decl, flag_finite_math_only)
|
|
|
407 || opt_for_fn (callee->decl, flag_signaling_nans)
|
|
|
408 != opt_for_fn (to->decl, flag_signaling_nans)
|
|
|
409 || opt_for_fn (callee->decl, flag_cx_limited_range)
|
|
|
410 != opt_for_fn (to->decl, flag_cx_limited_range)
|
|
|
411 || opt_for_fn (callee->decl, flag_signed_zeros)
|
|
|
412 != opt_for_fn (to->decl, flag_signed_zeros)
|
|
|
413 || opt_for_fn (callee->decl, flag_associative_math)
|
|
|
414 != opt_for_fn (to->decl, flag_associative_math)
|
|
|
415 || opt_for_fn (callee->decl, flag_reciprocal_math)
|
|
|
416 != opt_for_fn (to->decl, flag_reciprocal_math)
|
|
|
417 || opt_for_fn (callee->decl, flag_fp_int_builtin_inexact)
|
|
|
418 != opt_for_fn (to->decl, flag_fp_int_builtin_inexact)
|
|
|
419 || opt_for_fn (callee->decl, flag_errno_math)
|
|
|
420 != opt_for_fn (to->decl, flag_errno_math))
|
|
|
421 {
|
|
|
422 struct gcc_options opts = global_options;
|
|
|
423
|
|
|
424 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
425 opts.x_flag_rounding_math
|
|
|
426 = opt_for_fn (callee->decl, flag_rounding_math);
|
|
|
427 opts.x_flag_trapping_math
|
|
|
428 = opt_for_fn (callee->decl, flag_trapping_math);
|
|
|
429 opts.x_flag_unsafe_math_optimizations
|
|
|
430 = opt_for_fn (callee->decl, flag_unsafe_math_optimizations);
|
|
|
431 opts.x_flag_finite_math_only
|
|
|
432 = opt_for_fn (callee->decl, flag_finite_math_only);
|
|
|
433 opts.x_flag_signaling_nans
|
|
|
434 = opt_for_fn (callee->decl, flag_signaling_nans);
|
|
|
435 opts.x_flag_cx_limited_range
|
|
|
436 = opt_for_fn (callee->decl, flag_cx_limited_range);
|
|
|
437 opts.x_flag_signed_zeros
|
|
|
438 = opt_for_fn (callee->decl, flag_signed_zeros);
|
|
|
439 opts.x_flag_associative_math
|
|
|
440 = opt_for_fn (callee->decl, flag_associative_math);
|
|
|
441 opts.x_flag_reciprocal_math
|
|
|
442 = opt_for_fn (callee->decl, flag_reciprocal_math);
|
|
|
443 opts.x_flag_fp_int_builtin_inexact
|
|
|
444 = opt_for_fn (callee->decl, flag_fp_int_builtin_inexact);
|
|
|
445 opts.x_flag_errno_math
|
|
|
446 = opt_for_fn (callee->decl, flag_errno_math);
|
|
|
447 if (dump_file)
|
|
|
448 fprintf (dump_file, "Copying FP flags from %s to %s\n",
|
|
|
449 callee->dump_name (), to->dump_name ());
|
|
|
450 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
451 = build_optimization_node (&opts);
|
|
|
452 reload_optimization_node = true;
|
|
|
453 }
|
|
|
454 }
|
|
|
455
|
|
|
456 /* Reload global optimization flags. */
|
|
|
457 if (reload_optimization_node && DECL_STRUCT_FUNCTION (to->decl) == cfun)
|
|
|
458 set_cfun (cfun, true);
|
|
|
459
|
|
|
460 /* If aliases are involved, redirect edge to the actual destination and
|
|
|
461 possibly remove the aliases. */
|
|
|
462 if (e->callee != callee)
|
|
|
463 {
|
|
|
464 struct cgraph_node *alias = e->callee, *next_alias;
|
|
|
465 e->redirect_callee (callee);
|
|
|
466 while (alias && alias != callee)
|
|
|
467 {
|
|
|
468 if (!alias->callers
|
|
|
469 && can_remove_node_now_p (alias,
|
|
|
470 !e->next_caller && !e->prev_caller ? e : NULL))
|
|
|
471 {
|
|
|
472 next_alias = alias->get_alias_target ();
|
|
|
473 alias->remove ();
|
|
|
474 if (callee_removed)
|
|
|
475 *callee_removed = true;
|
|
|
476 alias = next_alias;
|
|
|
477 }
|
|
|
478 else
|
|
|
479 break;
|
|
|
480 }
|
|
|
481 }
|
|
|
482
|
|
131
|
483 clone_inlined_nodes (e, true, update_original, overall_size);
|
|
111
|
484
|
|
145
|
485 gcc_assert (curr->callee->inlined_to == to);
|
|
111
|
486
|
|
145
|
487 old_size = ipa_size_summaries->get (to)->size;
|
|
111
|
488 ipa_merge_fn_summary_after_inlining (e);
|
|
|
489 if (e->in_polymorphic_cdtor)
|
|
|
490 mark_all_inlined_calls_cdtor (e->callee);
|
|
|
491 if (opt_for_fn (e->caller->decl, optimize))
|
|
|
492 new_edges_found = ipa_propagate_indirect_call_infos (curr, new_edges);
|
|
145
|
493 bool removed_p = check_speculations (e->callee, new_edges);
|
|
111
|
494 if (update_overall_summary)
|
|
145
|
495 ipa_update_overall_fn_summary (to, new_edges_found || removed_p);
|
|
111
|
496 else
|
|
|
497 /* Update self size by the estimate so overall function growth limits
|
|
|
498 work for further inlining into this function. Before inlining
|
|
|
499 the function we inlined to again we expect the caller to update
|
|
|
500 the overall summary. */
|
|
145
|
501 ipa_size_summaries->get (to)->size += estimated_growth;
|
|
|
502 new_size = ipa_size_summaries->get (to)->size;
|
|
111
|
503
|
|
|
504 if (callee->calls_comdat_local)
|
|
|
505 to->calls_comdat_local = true;
|
|
145
|
506 else if (to->calls_comdat_local && comdat_local)
|
|
111
|
507 {
|
|
|
508 struct cgraph_edge *se = to->callees;
|
|
|
509 for (; se; se = se->next_callee)
|
|
|
510 if (se->inline_failed && se->callee->comdat_local_p ())
|
|
|
511 break;
|
|
|
512 if (se == NULL)
|
|
|
513 to->calls_comdat_local = false;
|
|
|
514 }
|
|
|
515
|
|
|
516 /* FIXME: This assert suffers from roundoff errors, disable it for GCC 5
|
|
|
517 and revisit it after conversion to sreals in GCC 6.
|
|
|
518 See PR 65654. */
|
|
|
519 #if 0
|
|
|
520 /* Verify that estimated growth match real growth. Allow off-by-one
|
|
|
521 error due to ipa_fn_summary::size_scale roudoff errors. */
|
|
|
522 gcc_assert (!update_overall_summary || !overall_size || new_edges_found
|
|
|
523 || abs (estimated_growth - (new_size - old_size)) <= 1
|
|
|
524 || speculation_removed
|
|
|
525 /* FIXME: a hack. Edges with false predicate are accounted
|
|
|
526 wrong, we should remove them from callgraph. */
|
|
|
527 || predicated);
|
|
|
528 #endif
|
|
|
529
|
|
|
530 /* Account the change of overall unit size; external functions will be
|
|
|
531 removed and are thus not accounted. */
|
|
|
532 if (overall_size && inline_account_function_p (to))
|
|
|
533 *overall_size += new_size - old_size;
|
|
|
534 ncalls_inlined++;
|
|
|
535
|
|
|
536 /* This must happen after ipa_merge_fn_summary_after_inlining that rely on jump
|
|
|
537 functions of callee to not be updated. */
|
|
|
538 return new_edges_found;
|
|
|
539 }
|
|
|
540
|
|
|
541
|
|
|
542 /* Copy function body of NODE and redirect all inline clones to it.
|
|
|
543 This is done before inline plan is applied to NODE when there are
|
|
|
544 still some inline clones if it.
|
|
|
545
|
|
|
546 This is necessary because inline decisions are not really transitive
|
|
|
547 and the other inline clones may have different bodies. */
|
|
|
548
|
|
|
549 static struct cgraph_node *
|
|
|
550 save_inline_function_body (struct cgraph_node *node)
|
|
|
551 {
|
|
|
552 struct cgraph_node *first_clone, *n;
|
|
|
553
|
|
|
554 if (dump_file)
|
|
|
555 fprintf (dump_file, "\nSaving body of %s for later reuse\n",
|
|
145
|
556 node->dump_name ());
|
|
111
|
557
|
|
|
558 gcc_assert (node == cgraph_node::get (node->decl));
|
|
|
559
|
|
|
560 /* first_clone will be turned into real function. */
|
|
|
561 first_clone = node->clones;
|
|
|
562
|
|
|
563 /* Arrange first clone to not be thunk as those do not have bodies. */
|
|
|
564 if (first_clone->thunk.thunk_p)
|
|
|
565 {
|
|
|
566 while (first_clone->thunk.thunk_p)
|
|
|
567 first_clone = first_clone->next_sibling_clone;
|
|
|
568 first_clone->prev_sibling_clone->next_sibling_clone
|
|
|
569 = first_clone->next_sibling_clone;
|
|
|
570 if (first_clone->next_sibling_clone)
|
|
|
571 first_clone->next_sibling_clone->prev_sibling_clone
|
|
|
572 = first_clone->prev_sibling_clone;
|
|
|
573 first_clone->next_sibling_clone = node->clones;
|
|
|
574 first_clone->prev_sibling_clone = NULL;
|
|
|
575 node->clones->prev_sibling_clone = first_clone;
|
|
|
576 node->clones = first_clone;
|
|
|
577 }
|
|
|
578 first_clone->decl = copy_node (node->decl);
|
|
|
579 first_clone->decl->decl_with_vis.symtab_node = first_clone;
|
|
|
580 gcc_assert (first_clone == cgraph_node::get (first_clone->decl));
|
|
|
581
|
|
|
582 /* Now reshape the clone tree, so all other clones descends from
|
|
|
583 first_clone. */
|
|
|
584 if (first_clone->next_sibling_clone)
|
|
|
585 {
|
|
|
586 for (n = first_clone->next_sibling_clone; n->next_sibling_clone;
|
|
|
587 n = n->next_sibling_clone)
|
|
|
588 n->clone_of = first_clone;
|
|
|
589 n->clone_of = first_clone;
|
|
|
590 n->next_sibling_clone = first_clone->clones;
|
|
|
591 if (first_clone->clones)
|
|
|
592 first_clone->clones->prev_sibling_clone = n;
|
|
|
593 first_clone->clones = first_clone->next_sibling_clone;
|
|
|
594 first_clone->next_sibling_clone->prev_sibling_clone = NULL;
|
|
|
595 first_clone->next_sibling_clone = NULL;
|
|
|
596 gcc_assert (!first_clone->prev_sibling_clone);
|
|
|
597 }
|
|
|
598 first_clone->clone_of = NULL;
|
|
|
599
|
|
|
600 /* Now node in question has no clones. */
|
|
|
601 node->clones = NULL;
|
|
|
602
|
|
|
603 /* Inline clones share decl with the function they are cloned
|
|
|
604 from. Walk the whole clone tree and redirect them all to the
|
|
|
605 new decl. */
|
|
|
606 if (first_clone->clones)
|
|
|
607 for (n = first_clone->clones; n != first_clone;)
|
|
|
608 {
|
|
|
609 gcc_assert (n->decl == node->decl);
|
|
|
610 n->decl = first_clone->decl;
|
|
|
611 if (n->clones)
|
|
|
612 n = n->clones;
|
|
|
613 else if (n->next_sibling_clone)
|
|
|
614 n = n->next_sibling_clone;
|
|
|
615 else
|
|
|
616 {
|
|
|
617 while (n != first_clone && !n->next_sibling_clone)
|
|
|
618 n = n->clone_of;
|
|
|
619 if (n != first_clone)
|
|
|
620 n = n->next_sibling_clone;
|
|
|
621 }
|
|
|
622 }
|
|
|
623
|
|
|
624 /* Copy the OLD_VERSION_NODE function tree to the new version. */
|
|
|
625 tree_function_versioning (node->decl, first_clone->decl,
|
|
145
|
626 NULL, NULL, true, NULL, NULL);
|
|
111
|
627
|
|
|
628 /* The function will be short lived and removed after we inline all the clones,
|
|
|
629 but make it internal so we won't confuse ourself. */
|
|
|
630 DECL_EXTERNAL (first_clone->decl) = 0;
|
|
|
631 TREE_PUBLIC (first_clone->decl) = 0;
|
|
|
632 DECL_COMDAT (first_clone->decl) = 0;
|
|
|
633 first_clone->ipa_transforms_to_apply.release ();
|
|
|
634
|
|
|
635 /* When doing recursive inlining, the clone may become unnecessary.
|
|
|
636 This is possible i.e. in the case when the recursive function is proved to be
|
|
|
637 non-throwing and the recursion happens only in the EH landing pad.
|
|
145
|
638 We cannot remove the clone until we are done with saving the body.
|
|
111
|
639 Remove it now. */
|
|
|
640 if (!first_clone->callers)
|
|
|
641 {
|
|
|
642 first_clone->remove_symbol_and_inline_clones ();
|
|
|
643 first_clone = NULL;
|
|
|
644 }
|
|
|
645 else if (flag_checking)
|
|
|
646 first_clone->verify ();
|
|
|
647
|
|
|
648 return first_clone;
|
|
|
649 }
|
|
|
650
|
|
|
651 /* Return true when function body of DECL still needs to be kept around
|
|
|
652 for later re-use. */
|
|
|
653 static bool
|
|
|
654 preserve_function_body_p (struct cgraph_node *node)
|
|
|
655 {
|
|
|
656 gcc_assert (symtab->global_info_ready);
|
|
|
657 gcc_assert (!node->alias && !node->thunk.thunk_p);
|
|
|
658
|
|
|
659 /* Look if there is any non-thunk clone around. */
|
|
|
660 for (node = node->clones; node; node = node->next_sibling_clone)
|
|
|
661 if (!node->thunk.thunk_p)
|
|
|
662 return true;
|
|
|
663 return false;
|
|
|
664 }
|
|
|
665
|
|
|
666 /* Apply inline plan to function. */
|
|
|
667
|
|
|
668 unsigned int
|
|
|
669 inline_transform (struct cgraph_node *node)
|
|
|
670 {
|
|
|
671 unsigned int todo = 0;
|
|
|
672 struct cgraph_edge *e, *next;
|
|
|
673 bool has_inline = false;
|
|
|
674
|
|
|
675 /* FIXME: Currently the pass manager is adding inline transform more than
|
|
|
676 once to some clones. This needs revisiting after WPA cleanups. */
|
|
|
677 if (cfun->after_inlining)
|
|
|
678 return 0;
|
|
|
679
|
|
|
680 /* We might need the body of this function so that we can expand
|
|
|
681 it inline somewhere else. */
|
|
|
682 if (preserve_function_body_p (node))
|
|
|
683 save_inline_function_body (node);
|
|
|
684
|
|
145
|
685 profile_count num = node->count;
|
|
|
686 profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
|
687 bool scale = num.initialized_p () && !(num == den);
|
|
|
688 if (scale)
|
|
|
689 {
|
|
|
690 profile_count::adjust_for_ipa_scaling (&num, &den);
|
|
|
691 if (dump_file)
|
|
|
692 {
|
|
|
693 fprintf (dump_file, "Applying count scale ");
|
|
|
694 num.dump (dump_file);
|
|
|
695 fprintf (dump_file, "/");
|
|
|
696 den.dump (dump_file);
|
|
|
697 fprintf (dump_file, "\n");
|
|
|
698 }
|
|
|
699
|
|
|
700 basic_block bb;
|
|
|
701 cfun->cfg->count_max = profile_count::uninitialized ();
|
|
|
702 FOR_ALL_BB_FN (bb, cfun)
|
|
|
703 {
|
|
|
704 bb->count = bb->count.apply_scale (num, den);
|
|
|
705 cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count);
|
|
|
706 }
|
|
|
707 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
|
|
|
708 }
|
|
|
709
|
|
111
|
710 for (e = node->callees; e; e = next)
|
|
|
711 {
|
|
|
712 if (!e->inline_failed)
|
|
|
713 has_inline = true;
|
|
|
714 next = e->next_callee;
|
|
145
|
715 cgraph_edge::redirect_call_stmt_to_callee (e);
|
|
111
|
716 }
|
|
|
717 node->remove_all_references ();
|
|
|
718
|
|
|
719 timevar_push (TV_INTEGRATION);
|
|
|
720 if (node->callees && (opt_for_fn (node->decl, optimize) || has_inline))
|
|
|
721 {
|
|
|
722 todo = optimize_inline_calls (current_function_decl);
|
|
145
|
723 }
|
|
111
|
724 timevar_pop (TV_INTEGRATION);
|
|
|
725
|
|
|
726 cfun->always_inline_functions_inlined = true;
|
|
|
727 cfun->after_inlining = true;
|
|
|
728 todo |= execute_fixup_cfg ();
|
|
|
729
|
|
|
730 if (!(todo & TODO_update_ssa_any))
|
|
|
731 /* Redirecting edges might lead to a need for vops to be recomputed. */
|
|
|
732 todo |= TODO_update_ssa_only_virtuals;
|
|
|
733
|
|
|
734 return todo;
|
|
|
735 }
|
