|
111
|
1 /* Callgraph transformations to handle inlining
|
|
131
|
2 Copyright (C) 2003-2018 Free Software Foundation, Inc.
|
|
111
|
3 Contributed by Jan Hubicka
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
|
21 /* The inline decisions are stored in callgraph in "inline plan" and
|
|
|
22 applied later.
|
|
|
23
|
|
|
24 To mark given call inline, use inline_call function.
|
|
|
25 The function marks the edge inlinable and, if necessary, produces
|
|
|
26 virtual clone in the callgraph representing the new copy of callee's
|
|
|
27 function body.
|
|
|
28
|
|
|
29 The inline plan is applied on given function body by inline_transform. */
|
|
|
30
|
|
|
31 #include "config.h"
|
|
|
32 #include "system.h"
|
|
|
33 #include "coretypes.h"
|
|
|
34 #include "tm.h"
|
|
|
35 #include "function.h"
|
|
|
36 #include "tree.h"
|
|
|
37 #include "alloc-pool.h"
|
|
|
38 #include "tree-pass.h"
|
|
|
39 #include "cgraph.h"
|
|
|
40 #include "tree-cfg.h"
|
|
|
41 #include "symbol-summary.h"
|
|
|
42 #include "tree-vrp.h"
|
|
|
43 #include "ipa-prop.h"
|
|
|
44 #include "ipa-fnsummary.h"
|
|
|
45 #include "ipa-inline.h"
|
|
|
46 #include "tree-inline.h"
|
|
|
47 #include "function.h"
|
|
|
48 #include "cfg.h"
|
|
|
49 #include "basic-block.h"
|
|
|
50
|
|
|
51 int ncalls_inlined;
|
|
|
52 int nfunctions_inlined;
|
|
|
53
|
|
131
|
54 /* Scale counts of NODE edges by NUM/DEN. */
|
|
111
|
55
|
|
|
56 static void
|
|
131
|
57 update_noncloned_counts (struct cgraph_node *node,
|
|
|
58 profile_count num, profile_count den)
|
|
111
|
59 {
|
|
|
60 struct cgraph_edge *e;
|
|
|
61
|
|
131
|
62 profile_count::adjust_for_ipa_scaling (&num, &den);
|
|
|
63
|
|
111
|
64 for (e = node->callees; e; e = e->next_callee)
|
|
|
65 {
|
|
|
66 if (!e->inline_failed)
|
|
131
|
67 update_noncloned_counts (e->callee, num, den);
|
|
|
68 e->count = e->count.apply_scale (num, den);
|
|
111
|
69 }
|
|
|
70 for (e = node->indirect_calls; e; e = e->next_callee)
|
|
131
|
71 e->count = e->count.apply_scale (num, den);
|
|
|
72 node->count = node->count.apply_scale (num, den);
|
|
111
|
73 }
|
|
|
74
|
|
|
75 /* We removed or are going to remove the last call to NODE.
|
|
|
76 Return true if we can and want proactively remove the NODE now.
|
|
|
77 This is important to do, since we want inliner to know when offline
|
|
|
78 copy of function was removed. */
|
|
|
79
|
|
|
80 static bool
|
|
|
81 can_remove_node_now_p_1 (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
82 {
|
|
|
83 ipa_ref *ref;
|
|
|
84
|
|
|
85 FOR_EACH_ALIAS (node, ref)
|
|
|
86 {
|
|
|
87 cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
|
|
|
88 if ((alias->callers && alias->callers != e)
|
|
|
89 || !can_remove_node_now_p_1 (alias, e))
|
|
|
90 return false;
|
|
|
91 }
|
|
|
92 /* FIXME: When address is taken of DECL_EXTERNAL function we still
|
|
|
93 can remove its offline copy, but we would need to keep unanalyzed node in
|
|
|
94 the callgraph so references can point to it.
|
|
|
95
|
|
|
96 Also for comdat group we can ignore references inside a group as we
|
|
|
97 want to prove the group as a whole to be dead. */
|
|
|
98 return (!node->address_taken
|
|
|
99 && node->can_remove_if_no_direct_calls_and_refs_p ()
|
|
|
100 /* Inlining might enable more devirtualizing, so we want to remove
|
|
|
101 those only after all devirtualizable virtual calls are processed.
|
|
|
102 Lacking may edges in callgraph we just preserve them post
|
|
|
103 inlining. */
|
|
|
104 && (!DECL_VIRTUAL_P (node->decl)
|
|
|
105 || !opt_for_fn (node->decl, flag_devirtualize))
|
|
|
106 /* During early inlining some unanalyzed cgraph nodes might be in the
|
|
|
107 callgraph and they might reffer the function in question. */
|
|
|
108 && !cgraph_new_nodes.exists ());
|
|
|
109 }
|
|
|
110
|
|
|
111 /* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
|
|
|
112 Verify that the NODE can be removed from unit and if it is contained in comdat
|
|
|
113 group that the whole comdat group is removable. */
|
|
|
114
|
|
|
115 static bool
|
|
|
116 can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
|
|
|
117 {
|
|
|
118 struct cgraph_node *next;
|
|
|
119 if (!can_remove_node_now_p_1 (node, e))
|
|
|
120 return false;
|
|
|
121
|
|
|
122 /* When we see same comdat group, we need to be sure that all
|
|
|
123 items can be removed. */
|
|
|
124 if (!node->same_comdat_group || !node->externally_visible)
|
|
|
125 return true;
|
|
|
126 for (next = dyn_cast<cgraph_node *> (node->same_comdat_group);
|
|
|
127 next != node; next = dyn_cast<cgraph_node *> (next->same_comdat_group))
|
|
|
128 {
|
|
|
129 if (next->alias)
|
|
|
130 continue;
|
|
|
131 if ((next->callers && next->callers != e)
|
|
|
132 || !can_remove_node_now_p_1 (next, e))
|
|
|
133 return false;
|
|
|
134 }
|
|
|
135 return true;
|
|
|
136 }
|
|
|
137
|
|
|
138 /* Return true if NODE is a master clone with non-inline clones. */
|
|
|
139
|
|
|
140 static bool
|
|
|
141 master_clone_with_noninline_clones_p (struct cgraph_node *node)
|
|
|
142 {
|
|
|
143 if (node->clone_of)
|
|
|
144 return false;
|
|
|
145
|
|
|
146 for (struct cgraph_node *n = node->clones; n; n = n->next_sibling_clone)
|
|
|
147 if (n->decl != node->decl)
|
|
|
148 return true;
|
|
|
149
|
|
|
150 return false;
|
|
|
151 }
|
|
|
152
|
|
|
153 /* E is expected to be an edge being inlined. Clone destination node of
|
|
|
154 the edge and redirect it to the new clone.
|
|
|
155 DUPLICATE is used for bookkeeping on whether we are actually creating new
|
|
|
156 clones or re-using node originally representing out-of-line function call.
|
|
|
157 By default the offline copy is removed, when it appears dead after inlining.
|
|
|
158 UPDATE_ORIGINAL prevents this transformation.
|
|
|
159 If OVERALL_SIZE is non-NULL, the size is updated to reflect the
|
|
131
|
160 transformation. */
|
|
111
|
161
|
|
|
162 void
|
|
|
163 clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
|
|
131
|
164 bool update_original, int *overall_size)
|
|
111
|
165 {
|
|
|
166 struct cgraph_node *inlining_into;
|
|
|
167 struct cgraph_edge *next;
|
|
|
168
|
|
|
169 if (e->caller->global.inlined_to)
|
|
|
170 inlining_into = e->caller->global.inlined_to;
|
|
|
171 else
|
|
|
172 inlining_into = e->caller;
|
|
|
173
|
|
|
174 if (duplicate)
|
|
|
175 {
|
|
|
176 /* We may eliminate the need for out-of-line copy to be output.
|
|
|
177 In that case just go ahead and re-use it. This is not just an
|
|
|
178 memory optimization. Making offline copy of fuction disappear
|
|
|
179 from the program will improve future decisions on inlining. */
|
|
|
180 if (!e->callee->callers->next_caller
|
|
|
181 /* Recursive inlining never wants the master clone to
|
|
|
182 be overwritten. */
|
|
|
183 && update_original
|
|
|
184 && can_remove_node_now_p (e->callee, e)
|
|
|
185 /* We cannot overwrite a master clone with non-inline clones
|
|
|
186 until after these clones are materialized. */
|
|
|
187 && !master_clone_with_noninline_clones_p (e->callee))
|
|
|
188 {
|
|
|
189 /* TODO: When callee is in a comdat group, we could remove all of it,
|
|
|
190 including all inline clones inlined into it. That would however
|
|
|
191 need small function inlining to register edge removal hook to
|
|
|
192 maintain the priority queue.
|
|
|
193
|
|
|
194 For now we keep the ohter functions in the group in program until
|
|
|
195 cgraph_remove_unreachable_functions gets rid of them. */
|
|
|
196 gcc_assert (!e->callee->global.inlined_to);
|
|
|
197 e->callee->remove_from_same_comdat_group ();
|
|
|
198 if (e->callee->definition
|
|
|
199 && inline_account_function_p (e->callee))
|
|
|
200 {
|
|
|
201 gcc_assert (!e->callee->alias);
|
|
|
202 if (overall_size)
|
|
131
|
203 *overall_size -= ipa_fn_summaries->get (e->callee)->size;
|
|
111
|
204 nfunctions_inlined++;
|
|
|
205 }
|
|
|
206 duplicate = false;
|
|
|
207 e->callee->externally_visible = false;
|
|
131
|
208 update_noncloned_counts (e->callee, e->count, e->callee->count);
|
|
111
|
209
|
|
|
210 dump_callgraph_transformation (e->callee, inlining_into,
|
|
|
211 "inlining to");
|
|
|
212 }
|
|
|
213 else
|
|
|
214 {
|
|
|
215 struct cgraph_node *n;
|
|
|
216
|
|
|
217 n = e->callee->create_clone (e->callee->decl,
|
|
131
|
218 e->count,
|
|
111
|
219 update_original, vNULL, true,
|
|
|
220 inlining_into,
|
|
|
221 NULL);
|
|
|
222 n->used_as_abstract_origin = e->callee->used_as_abstract_origin;
|
|
|
223 e->redirect_callee (n);
|
|
|
224 }
|
|
|
225 }
|
|
|
226 else
|
|
|
227 e->callee->remove_from_same_comdat_group ();
|
|
|
228
|
|
|
229 e->callee->global.inlined_to = inlining_into;
|
|
|
230
|
|
|
231 /* Recursively clone all bodies. */
|
|
|
232 for (e = e->callee->callees; e; e = next)
|
|
|
233 {
|
|
|
234 next = e->next_callee;
|
|
|
235 if (!e->inline_failed)
|
|
131
|
236 clone_inlined_nodes (e, duplicate, update_original, overall_size);
|
|
111
|
237 }
|
|
|
238 }
|
|
|
239
|
|
|
240 /* Check all speculations in N and resolve them if they seems useless. */
|
|
|
241
|
|
|
242 static bool
|
|
|
243 check_speculations (cgraph_node *n)
|
|
|
244 {
|
|
|
245 bool speculation_removed = false;
|
|
|
246 cgraph_edge *next;
|
|
|
247
|
|
|
248 for (cgraph_edge *e = n->callees; e; e = next)
|
|
|
249 {
|
|
|
250 next = e->next_callee;
|
|
|
251 if (e->speculative && !speculation_useful_p (e, true))
|
|
|
252 {
|
|
|
253 e->resolve_speculation (NULL);
|
|
|
254 speculation_removed = true;
|
|
|
255 }
|
|
|
256 else if (!e->inline_failed)
|
|
|
257 speculation_removed |= check_speculations (e->callee);
|
|
|
258 }
|
|
|
259 return speculation_removed;
|
|
|
260 }
|
|
|
261
|
|
|
262 /* Mark all call graph edges coming out of NODE and all nodes that have been
|
|
|
263 inlined to it as in_polymorphic_cdtor. */
|
|
|
264
|
|
|
265 static void
|
|
|
266 mark_all_inlined_calls_cdtor (cgraph_node *node)
|
|
|
267 {
|
|
|
268 for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
|
|
|
269 {
|
|
|
270 cs->in_polymorphic_cdtor = true;
|
|
|
271 if (!cs->inline_failed)
|
|
|
272 mark_all_inlined_calls_cdtor (cs->callee);
|
|
|
273 }
|
|
|
274 for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
|
|
|
275 cs->in_polymorphic_cdtor = true;
|
|
|
276 }
|
|
|
277
|
|
|
278
|
|
|
279 /* Mark edge E as inlined and update callgraph accordingly. UPDATE_ORIGINAL
|
|
|
280 specify whether profile of original function should be updated. If any new
|
|
|
281 indirect edges are discovered in the process, add them to NEW_EDGES, unless
|
|
|
282 it is NULL. If UPDATE_OVERALL_SUMMARY is false, do not bother to recompute overall
|
|
|
283 size of caller after inlining. Caller is required to eventually do it via
|
|
|
284 ipa_update_overall_fn_summary.
|
|
|
285 If callee_removed is non-NULL, set it to true if we removed callee node.
|
|
|
286
|
|
|
287 Return true iff any new callgraph edges were discovered as a
|
|
|
288 result of inlining. */
|
|
|
289
|
|
|
290 bool
|
|
|
291 inline_call (struct cgraph_edge *e, bool update_original,
|
|
|
292 vec<cgraph_edge *> *new_edges,
|
|
|
293 int *overall_size, bool update_overall_summary,
|
|
|
294 bool *callee_removed)
|
|
|
295 {
|
|
|
296 int old_size = 0, new_size = 0;
|
|
|
297 struct cgraph_node *to = NULL;
|
|
|
298 struct cgraph_edge *curr = e;
|
|
|
299 struct cgraph_node *callee = e->callee->ultimate_alias_target ();
|
|
|
300 bool new_edges_found = false;
|
|
|
301
|
|
|
302 int estimated_growth = 0;
|
|
|
303 if (! update_overall_summary)
|
|
|
304 estimated_growth = estimate_edge_growth (e);
|
|
|
305 /* This is used only for assert bellow. */
|
|
|
306 #if 0
|
|
|
307 bool predicated = inline_edge_summary (e)->predicate != NULL;
|
|
|
308 #endif
|
|
|
309
|
|
|
310 /* Don't inline inlined edges. */
|
|
|
311 gcc_assert (e->inline_failed);
|
|
|
312 /* Don't even think of inlining inline clone. */
|
|
|
313 gcc_assert (!callee->global.inlined_to);
|
|
|
314
|
|
|
315 to = e->caller;
|
|
|
316 if (to->global.inlined_to)
|
|
|
317 to = to->global.inlined_to;
|
|
|
318 if (to->thunk.thunk_p)
|
|
|
319 {
|
|
|
320 struct cgraph_node *target = to->callees->callee;
|
|
|
321 if (in_lto_p)
|
|
|
322 to->get_untransformed_body ();
|
|
|
323 to->expand_thunk (false, true);
|
|
|
324 /* When thunk is instrumented we may have multiple callees. */
|
|
|
325 for (e = to->callees; e && e->callee != target; e = e->next_callee)
|
|
|
326 ;
|
|
|
327 gcc_assert (e);
|
|
|
328 }
|
|
|
329
|
|
|
330
|
|
|
331 e->inline_failed = CIF_OK;
|
|
|
332 DECL_POSSIBLY_INLINED (callee->decl) = true;
|
|
|
333
|
|
|
334 if (DECL_FUNCTION_PERSONALITY (callee->decl))
|
|
|
335 DECL_FUNCTION_PERSONALITY (to->decl)
|
|
|
336 = DECL_FUNCTION_PERSONALITY (callee->decl);
|
|
|
337
|
|
|
338 bool reload_optimization_node = false;
|
|
|
339 if (!opt_for_fn (callee->decl, flag_strict_aliasing)
|
|
|
340 && opt_for_fn (to->decl, flag_strict_aliasing))
|
|
|
341 {
|
|
|
342 struct gcc_options opts = global_options;
|
|
|
343
|
|
|
344 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
345 opts.x_flag_strict_aliasing = false;
|
|
|
346 if (dump_file)
|
|
|
347 fprintf (dump_file, "Dropping flag_strict_aliasing on %s\n",
|
|
|
348 to->dump_name ());
|
|
|
349 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
350 = build_optimization_node (&opts);
|
|
|
351 reload_optimization_node = true;
|
|
|
352 }
|
|
|
353
|
|
|
354 ipa_fn_summary *caller_info = ipa_fn_summaries->get (to);
|
|
|
355 ipa_fn_summary *callee_info = ipa_fn_summaries->get (callee);
|
|
|
356 if (!caller_info->fp_expressions && callee_info->fp_expressions)
|
|
|
357 {
|
|
|
358 caller_info->fp_expressions = true;
|
|
|
359 if (opt_for_fn (callee->decl, flag_rounding_math)
|
|
|
360 != opt_for_fn (to->decl, flag_rounding_math)
|
|
|
361 || opt_for_fn (callee->decl, flag_trapping_math)
|
|
|
362 != opt_for_fn (to->decl, flag_trapping_math)
|
|
|
363 || opt_for_fn (callee->decl, flag_unsafe_math_optimizations)
|
|
|
364 != opt_for_fn (to->decl, flag_unsafe_math_optimizations)
|
|
|
365 || opt_for_fn (callee->decl, flag_finite_math_only)
|
|
|
366 != opt_for_fn (to->decl, flag_finite_math_only)
|
|
|
367 || opt_for_fn (callee->decl, flag_signaling_nans)
|
|
|
368 != opt_for_fn (to->decl, flag_signaling_nans)
|
|
|
369 || opt_for_fn (callee->decl, flag_cx_limited_range)
|
|
|
370 != opt_for_fn (to->decl, flag_cx_limited_range)
|
|
|
371 || opt_for_fn (callee->decl, flag_signed_zeros)
|
|
|
372 != opt_for_fn (to->decl, flag_signed_zeros)
|
|
|
373 || opt_for_fn (callee->decl, flag_associative_math)
|
|
|
374 != opt_for_fn (to->decl, flag_associative_math)
|
|
|
375 || opt_for_fn (callee->decl, flag_reciprocal_math)
|
|
|
376 != opt_for_fn (to->decl, flag_reciprocal_math)
|
|
|
377 || opt_for_fn (callee->decl, flag_fp_int_builtin_inexact)
|
|
|
378 != opt_for_fn (to->decl, flag_fp_int_builtin_inexact)
|
|
|
379 || opt_for_fn (callee->decl, flag_errno_math)
|
|
|
380 != opt_for_fn (to->decl, flag_errno_math))
|
|
|
381 {
|
|
|
382 struct gcc_options opts = global_options;
|
|
|
383
|
|
|
384 cl_optimization_restore (&opts, opts_for_fn (to->decl));
|
|
|
385 opts.x_flag_rounding_math
|
|
|
386 = opt_for_fn (callee->decl, flag_rounding_math);
|
|
|
387 opts.x_flag_trapping_math
|
|
|
388 = opt_for_fn (callee->decl, flag_trapping_math);
|
|
|
389 opts.x_flag_unsafe_math_optimizations
|
|
|
390 = opt_for_fn (callee->decl, flag_unsafe_math_optimizations);
|
|
|
391 opts.x_flag_finite_math_only
|
|
|
392 = opt_for_fn (callee->decl, flag_finite_math_only);
|
|
|
393 opts.x_flag_signaling_nans
|
|
|
394 = opt_for_fn (callee->decl, flag_signaling_nans);
|
|
|
395 opts.x_flag_cx_limited_range
|
|
|
396 = opt_for_fn (callee->decl, flag_cx_limited_range);
|
|
|
397 opts.x_flag_signed_zeros
|
|
|
398 = opt_for_fn (callee->decl, flag_signed_zeros);
|
|
|
399 opts.x_flag_associative_math
|
|
|
400 = opt_for_fn (callee->decl, flag_associative_math);
|
|
|
401 opts.x_flag_reciprocal_math
|
|
|
402 = opt_for_fn (callee->decl, flag_reciprocal_math);
|
|
|
403 opts.x_flag_fp_int_builtin_inexact
|
|
|
404 = opt_for_fn (callee->decl, flag_fp_int_builtin_inexact);
|
|
|
405 opts.x_flag_errno_math
|
|
|
406 = opt_for_fn (callee->decl, flag_errno_math);
|
|
|
407 if (dump_file)
|
|
|
408 fprintf (dump_file, "Copying FP flags from %s to %s\n",
|
|
|
409 callee->dump_name (), to->dump_name ());
|
|
|
410 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
|
|
|
411 = build_optimization_node (&opts);
|
|
|
412 reload_optimization_node = true;
|
|
|
413 }
|
|
|
414 }
|
|
|
415
|
|
|
416 /* Reload global optimization flags. */
|
|
|
417 if (reload_optimization_node && DECL_STRUCT_FUNCTION (to->decl) == cfun)
|
|
|
418 set_cfun (cfun, true);
|
|
|
419
|
|
|
420 /* If aliases are involved, redirect edge to the actual destination and
|
|
|
421 possibly remove the aliases. */
|
|
|
422 if (e->callee != callee)
|
|
|
423 {
|
|
|
424 struct cgraph_node *alias = e->callee, *next_alias;
|
|
|
425 e->redirect_callee (callee);
|
|
|
426 while (alias && alias != callee)
|
|
|
427 {
|
|
|
428 if (!alias->callers
|
|
|
429 && can_remove_node_now_p (alias,
|
|
|
430 !e->next_caller && !e->prev_caller ? e : NULL))
|
|
|
431 {
|
|
|
432 next_alias = alias->get_alias_target ();
|
|
|
433 alias->remove ();
|
|
|
434 if (callee_removed)
|
|
|
435 *callee_removed = true;
|
|
|
436 alias = next_alias;
|
|
|
437 }
|
|
|
438 else
|
|
|
439 break;
|
|
|
440 }
|
|
|
441 }
|
|
|
442
|
|
131
|
443 clone_inlined_nodes (e, true, update_original, overall_size);
|
|
111
|
444
|
|
|
445 gcc_assert (curr->callee->global.inlined_to == to);
|
|
|
446
|
|
|
447 old_size = ipa_fn_summaries->get (to)->size;
|
|
|
448 ipa_merge_fn_summary_after_inlining (e);
|
|
|
449 if (e->in_polymorphic_cdtor)
|
|
|
450 mark_all_inlined_calls_cdtor (e->callee);
|
|
|
451 if (opt_for_fn (e->caller->decl, optimize))
|
|
|
452 new_edges_found = ipa_propagate_indirect_call_infos (curr, new_edges);
|
|
|
453 check_speculations (e->callee);
|
|
|
454 if (update_overall_summary)
|
|
|
455 ipa_update_overall_fn_summary (to);
|
|
|
456 else
|
|
|
457 /* Update self size by the estimate so overall function growth limits
|
|
|
458 work for further inlining into this function. Before inlining
|
|
|
459 the function we inlined to again we expect the caller to update
|
|
|
460 the overall summary. */
|
|
|
461 ipa_fn_summaries->get (to)->size += estimated_growth;
|
|
|
462 new_size = ipa_fn_summaries->get (to)->size;
|
|
|
463
|
|
|
464 if (callee->calls_comdat_local)
|
|
|
465 to->calls_comdat_local = true;
|
|
|
466 else if (to->calls_comdat_local && callee->comdat_local_p ())
|
|
|
467 {
|
|
|
468 struct cgraph_edge *se = to->callees;
|
|
|
469 for (; se; se = se->next_callee)
|
|
|
470 if (se->inline_failed && se->callee->comdat_local_p ())
|
|
|
471 break;
|
|
|
472 if (se == NULL)
|
|
|
473 to->calls_comdat_local = false;
|
|
|
474 }
|
|
|
475
|
|
|
476 /* FIXME: This assert suffers from roundoff errors, disable it for GCC 5
|
|
|
477 and revisit it after conversion to sreals in GCC 6.
|
|
|
478 See PR 65654. */
|
|
|
479 #if 0
|
|
|
480 /* Verify that estimated growth match real growth. Allow off-by-one
|
|
|
481 error due to ipa_fn_summary::size_scale roudoff errors. */
|
|
|
482 gcc_assert (!update_overall_summary || !overall_size || new_edges_found
|
|
|
483 || abs (estimated_growth - (new_size - old_size)) <= 1
|
|
|
484 || speculation_removed
|
|
|
485 /* FIXME: a hack. Edges with false predicate are accounted
|
|
|
486 wrong, we should remove them from callgraph. */
|
|
|
487 || predicated);
|
|
|
488 #endif
|
|
|
489
|
|
|
490 /* Account the change of overall unit size; external functions will be
|
|
|
491 removed and are thus not accounted. */
|
|
|
492 if (overall_size && inline_account_function_p (to))
|
|
|
493 *overall_size += new_size - old_size;
|
|
|
494 ncalls_inlined++;
|
|
|
495
|
|
|
496 /* This must happen after ipa_merge_fn_summary_after_inlining that rely on jump
|
|
|
497 functions of callee to not be updated. */
|
|
|
498 return new_edges_found;
|
|
|
499 }
|
|
|
500
|
|
|
501
|
|
|
502 /* Copy function body of NODE and redirect all inline clones to it.
|
|
|
503 This is done before inline plan is applied to NODE when there are
|
|
|
504 still some inline clones if it.
|
|
|
505
|
|
|
506 This is necessary because inline decisions are not really transitive
|
|
|
507 and the other inline clones may have different bodies. */
|
|
|
508
|
|
|
509 static struct cgraph_node *
|
|
|
510 save_inline_function_body (struct cgraph_node *node)
|
|
|
511 {
|
|
|
512 struct cgraph_node *first_clone, *n;
|
|
|
513
|
|
|
514 if (dump_file)
|
|
|
515 fprintf (dump_file, "\nSaving body of %s for later reuse\n",
|
|
|
516 node->name ());
|
|
|
517
|
|
|
518 gcc_assert (node == cgraph_node::get (node->decl));
|
|
|
519
|
|
|
520 /* first_clone will be turned into real function. */
|
|
|
521 first_clone = node->clones;
|
|
|
522
|
|
|
523 /* Arrange first clone to not be thunk as those do not have bodies. */
|
|
|
524 if (first_clone->thunk.thunk_p)
|
|
|
525 {
|
|
|
526 while (first_clone->thunk.thunk_p)
|
|
|
527 first_clone = first_clone->next_sibling_clone;
|
|
|
528 first_clone->prev_sibling_clone->next_sibling_clone
|
|
|
529 = first_clone->next_sibling_clone;
|
|
|
530 if (first_clone->next_sibling_clone)
|
|
|
531 first_clone->next_sibling_clone->prev_sibling_clone
|
|
|
532 = first_clone->prev_sibling_clone;
|
|
|
533 first_clone->next_sibling_clone = node->clones;
|
|
|
534 first_clone->prev_sibling_clone = NULL;
|
|
|
535 node->clones->prev_sibling_clone = first_clone;
|
|
|
536 node->clones = first_clone;
|
|
|
537 }
|
|
|
538 first_clone->decl = copy_node (node->decl);
|
|
|
539 first_clone->decl->decl_with_vis.symtab_node = first_clone;
|
|
|
540 gcc_assert (first_clone == cgraph_node::get (first_clone->decl));
|
|
|
541
|
|
|
542 /* Now reshape the clone tree, so all other clones descends from
|
|
|
543 first_clone. */
|
|
|
544 if (first_clone->next_sibling_clone)
|
|
|
545 {
|
|
|
546 for (n = first_clone->next_sibling_clone; n->next_sibling_clone;
|
|
|
547 n = n->next_sibling_clone)
|
|
|
548 n->clone_of = first_clone;
|
|
|
549 n->clone_of = first_clone;
|
|
|
550 n->next_sibling_clone = first_clone->clones;
|
|
|
551 if (first_clone->clones)
|
|
|
552 first_clone->clones->prev_sibling_clone = n;
|
|
|
553 first_clone->clones = first_clone->next_sibling_clone;
|
|
|
554 first_clone->next_sibling_clone->prev_sibling_clone = NULL;
|
|
|
555 first_clone->next_sibling_clone = NULL;
|
|
|
556 gcc_assert (!first_clone->prev_sibling_clone);
|
|
|
557 }
|
|
|
558 first_clone->clone_of = NULL;
|
|
|
559
|
|
|
560 /* Now node in question has no clones. */
|
|
|
561 node->clones = NULL;
|
|
|
562
|
|
|
563 /* Inline clones share decl with the function they are cloned
|
|
|
564 from. Walk the whole clone tree and redirect them all to the
|
|
|
565 new decl. */
|
|
|
566 if (first_clone->clones)
|
|
|
567 for (n = first_clone->clones; n != first_clone;)
|
|
|
568 {
|
|
|
569 gcc_assert (n->decl == node->decl);
|
|
|
570 n->decl = first_clone->decl;
|
|
|
571 if (n->clones)
|
|
|
572 n = n->clones;
|
|
|
573 else if (n->next_sibling_clone)
|
|
|
574 n = n->next_sibling_clone;
|
|
|
575 else
|
|
|
576 {
|
|
|
577 while (n != first_clone && !n->next_sibling_clone)
|
|
|
578 n = n->clone_of;
|
|
|
579 if (n != first_clone)
|
|
|
580 n = n->next_sibling_clone;
|
|
|
581 }
|
|
|
582 }
|
|
|
583
|
|
|
584 /* Copy the OLD_VERSION_NODE function tree to the new version. */
|
|
|
585 tree_function_versioning (node->decl, first_clone->decl,
|
|
|
586 NULL, true, NULL, false,
|
|
|
587 NULL, NULL);
|
|
|
588
|
|
|
589 /* The function will be short lived and removed after we inline all the clones,
|
|
|
590 but make it internal so we won't confuse ourself. */
|
|
|
591 DECL_EXTERNAL (first_clone->decl) = 0;
|
|
|
592 TREE_PUBLIC (first_clone->decl) = 0;
|
|
|
593 DECL_COMDAT (first_clone->decl) = 0;
|
|
|
594 first_clone->ipa_transforms_to_apply.release ();
|
|
|
595
|
|
|
596 /* When doing recursive inlining, the clone may become unnecessary.
|
|
|
597 This is possible i.e. in the case when the recursive function is proved to be
|
|
|
598 non-throwing and the recursion happens only in the EH landing pad.
|
|
|
599 We can not remove the clone until we are done with saving the body.
|
|
|
600 Remove it now. */
|
|
|
601 if (!first_clone->callers)
|
|
|
602 {
|
|
|
603 first_clone->remove_symbol_and_inline_clones ();
|
|
|
604 first_clone = NULL;
|
|
|
605 }
|
|
|
606 else if (flag_checking)
|
|
|
607 first_clone->verify ();
|
|
|
608
|
|
|
609 return first_clone;
|
|
|
610 }
|
|
|
611
|
|
|
612 /* Return true when function body of DECL still needs to be kept around
|
|
|
613 for later re-use. */
|
|
|
614 static bool
|
|
|
615 preserve_function_body_p (struct cgraph_node *node)
|
|
|
616 {
|
|
|
617 gcc_assert (symtab->global_info_ready);
|
|
|
618 gcc_assert (!node->alias && !node->thunk.thunk_p);
|
|
|
619
|
|
|
620 /* Look if there is any non-thunk clone around. */
|
|
|
621 for (node = node->clones; node; node = node->next_sibling_clone)
|
|
|
622 if (!node->thunk.thunk_p)
|
|
|
623 return true;
|
|
|
624 return false;
|
|
|
625 }
|
|
|
626
|
|
|
627 /* Apply inline plan to function. */
|
|
|
628
|
|
|
629 unsigned int
|
|
|
630 inline_transform (struct cgraph_node *node)
|
|
|
631 {
|
|
|
632 unsigned int todo = 0;
|
|
|
633 struct cgraph_edge *e, *next;
|
|
|
634 bool has_inline = false;
|
|
|
635
|
|
|
636 /* FIXME: Currently the pass manager is adding inline transform more than
|
|
|
637 once to some clones. This needs revisiting after WPA cleanups. */
|
|
|
638 if (cfun->after_inlining)
|
|
|
639 return 0;
|
|
|
640
|
|
|
641 /* We might need the body of this function so that we can expand
|
|
|
642 it inline somewhere else. */
|
|
|
643 if (preserve_function_body_p (node))
|
|
|
644 save_inline_function_body (node);
|
|
|
645
|
|
|
646 for (e = node->callees; e; e = next)
|
|
|
647 {
|
|
|
648 if (!e->inline_failed)
|
|
|
649 has_inline = true;
|
|
|
650 next = e->next_callee;
|
|
|
651 e->redirect_call_stmt_to_callee ();
|
|
|
652 }
|
|
|
653 node->remove_all_references ();
|
|
|
654
|
|
|
655 timevar_push (TV_INTEGRATION);
|
|
|
656 if (node->callees && (opt_for_fn (node->decl, optimize) || has_inline))
|
|
|
657 {
|
|
|
658 profile_count num = node->count;
|
|
|
659 profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
131
|
660 bool scale = num.initialized_p () && !(num == den);
|
|
111
|
661 if (scale)
|
|
|
662 {
|
|
131
|
663 profile_count::adjust_for_ipa_scaling (&num, &den);
|
|
111
|
664 if (dump_file)
|
|
|
665 {
|
|
|
666 fprintf (dump_file, "Applying count scale ");
|
|
|
667 num.dump (dump_file);
|
|
|
668 fprintf (dump_file, "/");
|
|
|
669 den.dump (dump_file);
|
|
|
670 fprintf (dump_file, "\n");
|
|
|
671 }
|
|
|
672
|
|
|
673 basic_block bb;
|
|
131
|
674 cfun->cfg->count_max = profile_count::uninitialized ();
|
|
111
|
675 FOR_ALL_BB_FN (bb, cfun)
|
|
131
|
676 {
|
|
|
677 bb->count = bb->count.apply_scale (num, den);
|
|
|
678 cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count);
|
|
|
679 }
|
|
111
|
680 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
|
|
|
681 }
|
|
|
682 todo = optimize_inline_calls (current_function_decl);
|
|
|
683 }
|
|
|
684 timevar_pop (TV_INTEGRATION);
|
|
|
685
|
|
|
686 cfun->always_inline_functions_inlined = true;
|
|
|
687 cfun->after_inlining = true;
|
|
|
688 todo |= execute_fixup_cfg ();
|
|
|
689
|
|
|
690 if (!(todo & TODO_update_ssa_any))
|
|
|
691 /* Redirecting edges might lead to a need for vops to be recomputed. */
|
|
|
692 todo |= TODO_update_ssa_only_virtuals;
|
|
|
693
|
|
|
694 return todo;
|
|
|
695 }
|
