|
111
|
1 /* Callgraph clones
|
|
145
|
2 Copyright (C) 2003-2020 Free Software Foundation, Inc.
|
|
111
|
3 Contributed by Jan Hubicka
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
145
|
21 /* This module provide facilities for cloning functions. I.e. creating
|
|
111
|
22 new functions based on existing functions with simple modifications,
|
|
|
23 such as replacement of parameters.
|
|
|
24
|
|
|
25 To allow whole program optimization without actual presence of function
|
|
|
26 bodies, an additional infrastructure is provided for so-called virtual
|
|
|
27 clones
|
|
|
28
|
|
|
29 A virtual clone in the callgraph is a function that has no
|
|
|
30 associated body, just a description of how to create its body based
|
|
|
31 on a different function (which itself may be a virtual clone).
|
|
|
32
|
|
|
33 The description of function modifications includes adjustments to
|
|
|
34 the function's signature (which allows, for example, removing or
|
|
|
35 adding function arguments), substitutions to perform on the
|
|
|
36 function body, and, for inlined functions, a pointer to the
|
|
|
37 function that it will be inlined into.
|
|
|
38
|
|
|
39 It is also possible to redirect any edge of the callgraph from a
|
|
|
40 function to its virtual clone. This implies updating of the call
|
|
|
41 site to adjust for the new function signature.
|
|
|
42
|
|
|
43 Most of the transformations performed by inter-procedural
|
|
|
44 optimizations can be represented via virtual clones. For
|
|
|
45 instance, a constant propagation pass can produce a virtual clone
|
|
|
46 of the function which replaces one of its arguments by a
|
|
|
47 constant. The inliner can represent its decisions by producing a
|
|
|
48 clone of a function whose body will be later integrated into
|
|
|
49 a given function.
|
|
|
50
|
|
|
51 Using virtual clones, the program can be easily updated
|
|
|
52 during the Execute stage, solving most of pass interactions
|
|
|
53 problems that would otherwise occur during Transform.
|
|
|
54
|
|
|
55 Virtual clones are later materialized in the LTRANS stage and
|
|
|
56 turned into real functions. Passes executed after the virtual
|
|
|
57 clone were introduced also perform their Transform stage
|
|
|
58 on new functions, so for a pass there is no significant
|
|
|
59 difference between operating on a real function or a virtual
|
|
|
60 clone introduced before its Execute stage.
|
|
|
61
|
|
|
62 Optimization passes then work on virtual clones introduced before
|
|
|
63 their Execute stage as if they were real functions. The
|
|
|
64 only difference is that clones are not visible during the
|
|
|
65 Generate Summary stage. */
|
|
|
66
|
|
|
67 #include "config.h"
|
|
|
68 #include "system.h"
|
|
|
69 #include "coretypes.h"
|
|
|
70 #include "backend.h"
|
|
|
71 #include "target.h"
|
|
|
72 #include "rtl.h"
|
|
|
73 #include "tree.h"
|
|
|
74 #include "gimple.h"
|
|
|
75 #include "stringpool.h"
|
|
|
76 #include "cgraph.h"
|
|
|
77 #include "lto-streamer.h"
|
|
|
78 #include "tree-eh.h"
|
|
|
79 #include "tree-cfg.h"
|
|
|
80 #include "tree-inline.h"
|
|
|
81 #include "dumpfile.h"
|
|
|
82 #include "gimple-pretty-print.h"
|
|
145
|
83 #include "alloc-pool.h"
|
|
|
84 #include "symbol-summary.h"
|
|
|
85 #include "tree-vrp.h"
|
|
|
86 #include "ipa-prop.h"
|
|
|
87 #include "ipa-fnsummary.h"
|
|
111
|
88
|
|
|
89 /* Create clone of edge in the node N represented by CALL_EXPR
|
|
|
90 the callgraph. */
|
|
|
91
|
|
|
92 cgraph_edge *
|
|
|
93 cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
|
|
|
94 profile_count num, profile_count den,
|
|
131
|
95 bool update_original)
|
|
111
|
96 {
|
|
|
97 cgraph_edge *new_edge;
|
|
131
|
98 profile_count::adjust_for_ipa_scaling (&num, &den);
|
|
|
99 profile_count prof_count = count.apply_scale (num, den);
|
|
111
|
100
|
|
|
101 if (indirect_unknown_callee)
|
|
|
102 {
|
|
|
103 tree decl;
|
|
|
104
|
|
|
105 if (call_stmt && (decl = gimple_call_fndecl (call_stmt))
|
|
|
106 /* When the call is speculative, we need to resolve it
|
|
|
107 via cgraph_resolve_speculation and not here. */
|
|
|
108 && !speculative)
|
|
|
109 {
|
|
|
110 cgraph_node *callee = cgraph_node::get (decl);
|
|
|
111 gcc_checking_assert (callee);
|
|
145
|
112 new_edge = n->create_edge (callee, call_stmt, prof_count, true);
|
|
111
|
113 }
|
|
|
114 else
|
|
|
115 {
|
|
|
116 new_edge = n->create_indirect_edge (call_stmt,
|
|
|
117 indirect_info->ecf_flags,
|
|
145
|
118 prof_count, true);
|
|
111
|
119 *new_edge->indirect_info = *indirect_info;
|
|
|
120 }
|
|
|
121 }
|
|
|
122 else
|
|
|
123 {
|
|
145
|
124 new_edge = n->create_edge (callee, call_stmt, prof_count, true);
|
|
111
|
125 if (indirect_info)
|
|
|
126 {
|
|
|
127 new_edge->indirect_info
|
|
|
128 = ggc_cleared_alloc<cgraph_indirect_call_info> ();
|
|
|
129 *new_edge->indirect_info = *indirect_info;
|
|
|
130 }
|
|
|
131 }
|
|
|
132
|
|
|
133 new_edge->inline_failed = inline_failed;
|
|
|
134 new_edge->indirect_inlining_edge = indirect_inlining_edge;
|
|
145
|
135 if (!call_stmt)
|
|
|
136 new_edge->lto_stmt_uid = stmt_uid;
|
|
|
137 new_edge->speculative_id = speculative_id;
|
|
111
|
138 /* Clone flags that depend on call_stmt availability manually. */
|
|
|
139 new_edge->can_throw_external = can_throw_external;
|
|
|
140 new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p;
|
|
|
141 new_edge->speculative = speculative;
|
|
|
142 new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor;
|
|
131
|
143
|
|
|
144 /* Update IPA profile. Local profiles need no updating in original. */
|
|
111
|
145 if (update_original)
|
|
145
|
146 count = count.combine_with_ipa_count_within (count.ipa ()
|
|
|
147 - new_edge->count.ipa (),
|
|
|
148 caller->count);
|
|
111
|
149 symtab->call_edge_duplication_hooks (this, new_edge);
|
|
|
150 return new_edge;
|
|
|
151 }
|
|
|
152
|
|
|
153 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
|
|
|
154 clone or its thunk. */
|
|
|
155
|
|
|
156 static void
|
|
|
157 set_new_clone_decl_and_node_flags (cgraph_node *new_node)
|
|
|
158 {
|
|
|
159 DECL_EXTERNAL (new_node->decl) = 0;
|
|
|
160 TREE_PUBLIC (new_node->decl) = 0;
|
|
|
161 DECL_COMDAT (new_node->decl) = 0;
|
|
|
162 DECL_WEAK (new_node->decl) = 0;
|
|
|
163 DECL_VIRTUAL_P (new_node->decl) = 0;
|
|
|
164 DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0;
|
|
|
165 DECL_STATIC_DESTRUCTOR (new_node->decl) = 0;
|
|
145
|
166 DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0);
|
|
|
167 DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0);
|
|
111
|
168
|
|
|
169 new_node->externally_visible = 0;
|
|
145
|
170 new_node->local = 1;
|
|
111
|
171 new_node->lowered = true;
|
|
|
172 }
|
|
|
173
|
|
|
174 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
|
|
|
175 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
|
|
|
176 Function can return NODE if no thunk is necessary, which can happen when
|
|
|
177 thunk is this_adjusting but we are removing this parameter. */
|
|
|
178
|
|
|
179 static cgraph_node *
|
|
|
180 duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node)
|
|
|
181 {
|
|
|
182 cgraph_node *new_thunk, *thunk_of;
|
|
|
183 thunk_of = thunk->callees->callee->ultimate_alias_target ();
|
|
|
184
|
|
|
185 if (thunk_of->thunk.thunk_p)
|
|
|
186 node = duplicate_thunk_for_node (thunk_of, node);
|
|
|
187
|
|
|
188 if (!DECL_ARGUMENTS (thunk->decl))
|
|
|
189 thunk->get_untransformed_body ();
|
|
|
190
|
|
|
191 cgraph_edge *cs;
|
|
|
192 for (cs = node->callers; cs; cs = cs->next_caller)
|
|
|
193 if (cs->caller->thunk.thunk_p
|
|
131
|
194 && cs->caller->thunk.fixed_offset == thunk->thunk.fixed_offset
|
|
|
195 && cs->caller->thunk.virtual_value == thunk->thunk.virtual_value
|
|
|
196 && cs->caller->thunk.indirect_offset == thunk->thunk.indirect_offset
|
|
111
|
197 && cs->caller->thunk.this_adjusting == thunk->thunk.this_adjusting
|
|
131
|
198 && cs->caller->thunk.virtual_offset_p == thunk->thunk.virtual_offset_p)
|
|
111
|
199 return cs->caller;
|
|
|
200
|
|
|
201 tree new_decl;
|
|
145
|
202 if (node->clone.param_adjustments)
|
|
111
|
203 {
|
|
|
204 /* We do not need to duplicate this_adjusting thunks if we have removed
|
|
|
205 this. */
|
|
|
206 if (thunk->thunk.this_adjusting
|
|
145
|
207 && !node->clone.param_adjustments->first_param_intact_p ())
|
|
111
|
208 return node;
|
|
|
209
|
|
145
|
210 new_decl = copy_node (thunk->decl);
|
|
|
211 ipa_param_body_adjustments body_adj (node->clone.param_adjustments,
|
|
|
212 new_decl);
|
|
|
213 body_adj.modify_formal_parameters ();
|
|
111
|
214 }
|
|
145
|
215 else
|
|
|
216 new_decl = copy_node (thunk->decl);
|
|
111
|
217
|
|
|
218 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl));
|
|
|
219 gcc_checking_assert (!DECL_INITIAL (new_decl));
|
|
|
220 gcc_checking_assert (!DECL_RESULT (new_decl));
|
|
|
221 gcc_checking_assert (!DECL_RTL_SET_P (new_decl));
|
|
|
222
|
|
145
|
223 DECL_NAME (new_decl) = clone_function_name_numbered (thunk->decl,
|
|
|
224 "artificial_thunk");
|
|
111
|
225 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
|
|
|
226
|
|
145
|
227 /* We need to force DECL_IGNORED_P because the new thunk is created after
|
|
|
228 early debug was run. */
|
|
|
229 DECL_IGNORED_P (new_decl) = 1;
|
|
|
230
|
|
111
|
231 new_thunk = cgraph_node::create (new_decl);
|
|
|
232 set_new_clone_decl_and_node_flags (new_thunk);
|
|
|
233 new_thunk->definition = true;
|
|
145
|
234 new_thunk->can_change_signature = node->can_change_signature;
|
|
111
|
235 new_thunk->thunk = thunk->thunk;
|
|
|
236 new_thunk->unique_name = in_lto_p;
|
|
|
237 new_thunk->former_clone_of = thunk->decl;
|
|
145
|
238 new_thunk->clone.param_adjustments = node->clone.param_adjustments;
|
|
|
239 new_thunk->unit_id = thunk->unit_id;
|
|
|
240 new_thunk->merged_comdat = thunk->merged_comdat;
|
|
|
241 new_thunk->merged_extern_inline = thunk->merged_extern_inline;
|
|
111
|
242
|
|
131
|
243 cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count);
|
|
111
|
244 symtab->call_edge_duplication_hooks (thunk->callees, e);
|
|
|
245 symtab->call_cgraph_duplication_hooks (thunk, new_thunk);
|
|
|
246 return new_thunk;
|
|
|
247 }
|
|
|
248
|
|
|
249 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
|
|
|
250 one or more equivalent thunks for N and redirect E to the first in the
|
|
|
251 chain. Note that it is then necessary to call
|
|
|
252 n->expand_all_artificial_thunks once all callers are redirected. */
|
|
|
253
|
|
|
254 void
|
|
|
255 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n)
|
|
|
256 {
|
|
|
257 cgraph_node *orig_to = callee->ultimate_alias_target ();
|
|
|
258 if (orig_to->thunk.thunk_p)
|
|
|
259 n = duplicate_thunk_for_node (orig_to, n);
|
|
|
260
|
|
|
261 redirect_callee (n);
|
|
|
262 }
|
|
|
263
|
|
|
264 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
|
|
|
265 that were expanded. */
|
|
|
266
|
|
|
267 void
|
|
|
268 cgraph_node::expand_all_artificial_thunks ()
|
|
|
269 {
|
|
|
270 cgraph_edge *e;
|
|
|
271 for (e = callers; e;)
|
|
|
272 if (e->caller->thunk.thunk_p)
|
|
|
273 {
|
|
|
274 cgraph_node *thunk = e->caller;
|
|
|
275
|
|
|
276 e = e->next_caller;
|
|
|
277 if (thunk->expand_thunk (false, false))
|
|
|
278 {
|
|
|
279 thunk->thunk.thunk_p = false;
|
|
|
280 thunk->analyze ();
|
|
145
|
281 ipa_analyze_node (thunk);
|
|
|
282 inline_analyze_function (thunk);
|
|
111
|
283 }
|
|
|
284 thunk->expand_all_artificial_thunks ();
|
|
|
285 }
|
|
|
286 else
|
|
|
287 e = e->next_caller;
|
|
|
288 }
|
|
|
289
|
|
|
290 void
|
|
|
291 dump_callgraph_transformation (const cgraph_node *original,
|
|
|
292 const cgraph_node *clone,
|
|
|
293 const char *suffix)
|
|
|
294 {
|
|
|
295 if (symtab->ipa_clones_dump_file)
|
|
|
296 {
|
|
|
297 fprintf (symtab->ipa_clones_dump_file,
|
|
|
298 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
|
|
|
299 original->asm_name (), original->order,
|
|
|
300 DECL_SOURCE_FILE (original->decl),
|
|
|
301 DECL_SOURCE_LINE (original->decl),
|
|
|
302 DECL_SOURCE_COLUMN (original->decl), clone->asm_name (),
|
|
|
303 clone->order, DECL_SOURCE_FILE (clone->decl),
|
|
|
304 DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl),
|
|
|
305 suffix);
|
|
|
306
|
|
|
307 symtab->cloned_nodes.add (original);
|
|
|
308 symtab->cloned_nodes.add (clone);
|
|
|
309 }
|
|
|
310 }
|
|
|
311
|
|
145
|
312 /* Turn profile of N to local profile. */
|
|
|
313
|
|
|
314 static void
|
|
|
315 localize_profile (cgraph_node *n)
|
|
|
316 {
|
|
|
317 n->count = n->count.guessed_local ();
|
|
|
318 for (cgraph_edge *e = n->callees; e; e=e->next_callee)
|
|
|
319 {
|
|
|
320 e->count = e->count.guessed_local ();
|
|
|
321 if (!e->inline_failed)
|
|
|
322 localize_profile (e->callee);
|
|
|
323 }
|
|
|
324 for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee)
|
|
|
325 e->count = e->count.guessed_local ();
|
|
|
326 }
|
|
|
327
|
|
111
|
328 /* Create node representing clone of N executed COUNT times. Decrease
|
|
|
329 the execution counts from original node too.
|
|
|
330 The new clone will have decl set to DECL that may or may not be the same
|
|
|
331 as decl of N.
|
|
|
332
|
|
|
333 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
|
|
|
334 function's profile to reflect the fact that part of execution is handled
|
|
|
335 by node.
|
|
145
|
336 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
|
|
111
|
337 the new clone. Otherwise the caller is responsible for doing so later.
|
|
|
338
|
|
|
339 If the new node is being inlined into another one, NEW_INLINED_TO should be
|
|
|
340 the outline function the new one is (even indirectly) inlined to. All hooks
|
|
145
|
341 will see this in node's inlined_to, when invoked. Can be NULL if the
|
|
|
342 node is not inlined.
|
|
|
343
|
|
|
344 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
|
|
|
345 will be overwritten by the new structure. Otherwise the new node will
|
|
|
346 share parameter manipulation information with the original node. */
|
|
111
|
347
|
|
|
348 cgraph_node *
|
|
131
|
349 cgraph_node::create_clone (tree new_decl, profile_count prof_count,
|
|
111
|
350 bool update_original,
|
|
|
351 vec<cgraph_edge *> redirect_callers,
|
|
|
352 bool call_duplication_hook,
|
|
|
353 cgraph_node *new_inlined_to,
|
|
145
|
354 ipa_param_adjustments *param_adjustments,
|
|
|
355 const char *suffix)
|
|
111
|
356 {
|
|
|
357 cgraph_node *new_node = symtab->create_empty ();
|
|
|
358 cgraph_edge *e;
|
|
|
359 unsigned i;
|
|
131
|
360 profile_count old_count = count;
|
|
145
|
361 bool nonzero = count.ipa ().nonzero_p ();
|
|
111
|
362
|
|
|
363 if (new_inlined_to)
|
|
|
364 dump_callgraph_transformation (this, new_inlined_to, "inlining to");
|
|
|
365
|
|
131
|
366 /* When inlining we scale precisely to prof_count, when cloning we can
|
|
|
367 preserve local profile. */
|
|
|
368 if (!new_inlined_to)
|
|
|
369 prof_count = count.combine_with_ipa_count (prof_count);
|
|
111
|
370 new_node->count = prof_count;
|
|
131
|
371
|
|
|
372 /* Update IPA profile. Local profiles need no updating in original. */
|
|
|
373 if (update_original)
|
|
145
|
374 {
|
|
|
375 if (inlined_to)
|
|
|
376 count = count.combine_with_ipa_count_within (count.ipa ()
|
|
|
377 - prof_count.ipa (),
|
|
|
378 inlined_to->count);
|
|
|
379 else
|
|
|
380 count = count.combine_with_ipa_count (count.ipa () - prof_count.ipa ());
|
|
|
381 }
|
|
111
|
382 new_node->decl = new_decl;
|
|
|
383 new_node->register_symbol ();
|
|
|
384 new_node->origin = origin;
|
|
|
385 new_node->lto_file_data = lto_file_data;
|
|
|
386 if (new_node->origin)
|
|
|
387 {
|
|
|
388 new_node->next_nested = new_node->origin->nested;
|
|
|
389 new_node->origin->nested = new_node;
|
|
|
390 }
|
|
|
391 new_node->analyzed = analyzed;
|
|
|
392 new_node->definition = definition;
|
|
145
|
393 new_node->versionable = versionable;
|
|
|
394 new_node->can_change_signature = can_change_signature;
|
|
|
395 new_node->redefined_extern_inline = redefined_extern_inline;
|
|
|
396 new_node->tm_may_enter_irr = tm_may_enter_irr;
|
|
111
|
397 new_node->externally_visible = false;
|
|
|
398 new_node->no_reorder = no_reorder;
|
|
145
|
399 new_node->local = true;
|
|
|
400 new_node->inlined_to = new_inlined_to;
|
|
111
|
401 new_node->rtl = rtl;
|
|
|
402 new_node->frequency = frequency;
|
|
|
403 new_node->tp_first_run = tp_first_run;
|
|
|
404 new_node->tm_clone = tm_clone;
|
|
|
405 new_node->icf_merged = icf_merged;
|
|
|
406 new_node->merged_comdat = merged_comdat;
|
|
|
407 new_node->thunk = thunk;
|
|
145
|
408 new_node->unit_id = unit_id;
|
|
|
409 new_node->merged_comdat = merged_comdat;
|
|
|
410 new_node->merged_extern_inline = merged_extern_inline;
|
|
111
|
411
|
|
145
|
412 if (param_adjustments)
|
|
|
413 new_node->clone.param_adjustments = param_adjustments;
|
|
|
414 else
|
|
|
415 new_node->clone.param_adjustments = clone.param_adjustments;
|
|
111
|
416 new_node->clone.tree_map = NULL;
|
|
145
|
417 new_node->clone.performed_splits = vec_safe_copy (clone.performed_splits);
|
|
111
|
418 new_node->split_part = split_part;
|
|
|
419
|
|
|
420 FOR_EACH_VEC_ELT (redirect_callers, i, e)
|
|
|
421 {
|
|
|
422 /* Redirect calls to the old version node to point to its new
|
|
|
423 version. The only exception is when the edge was proved to
|
|
145
|
424 be unreachable during the cloning procedure. */
|
|
111
|
425 if (!e->callee
|
|
131
|
426 || !fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE))
|
|
111
|
427 e->redirect_callee_duplicating_thunks (new_node);
|
|
|
428 }
|
|
|
429 new_node->expand_all_artificial_thunks ();
|
|
|
430
|
|
|
431 for (e = callees;e; e=e->next_callee)
|
|
131
|
432 e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count,
|
|
|
433 update_original);
|
|
111
|
434
|
|
|
435 for (e = indirect_calls; e; e = e->next_callee)
|
|
|
436 e->clone (new_node, e->call_stmt, e->lto_stmt_uid,
|
|
131
|
437 new_node->count, old_count, update_original);
|
|
111
|
438 new_node->clone_references (this);
|
|
|
439
|
|
|
440 new_node->next_sibling_clone = clones;
|
|
|
441 if (clones)
|
|
|
442 clones->prev_sibling_clone = new_node;
|
|
|
443 clones = new_node;
|
|
|
444 new_node->clone_of = this;
|
|
|
445
|
|
|
446 if (call_duplication_hook)
|
|
|
447 symtab->call_cgraph_duplication_hooks (this, new_node);
|
|
145
|
448 /* With partial train run we do not want to assume that original's
|
|
|
449 count is zero whenever we redurect all executed edges to clone.
|
|
|
450 Simply drop profile to local one in this case. */
|
|
|
451 if (update_original
|
|
|
452 && opt_for_fn (decl, flag_profile_partial_training)
|
|
|
453 && nonzero
|
|
|
454 && count.ipa_p ()
|
|
|
455 && !count.ipa ().nonzero_p ()
|
|
|
456 && !inlined_to)
|
|
|
457 localize_profile (this);
|
|
111
|
458
|
|
|
459 if (!new_inlined_to)
|
|
|
460 dump_callgraph_transformation (this, new_node, suffix);
|
|
|
461
|
|
|
462 return new_node;
|
|
|
463 }
|
|
|
464
|
|
145
|
465 static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids;
|
|
111
|
466
|
|
145
|
467 /* Return a new assembler name for a clone of decl named NAME. Apart
|
|
|
468 from the string SUFFIX, the new name will end with a unique (for
|
|
|
469 each NAME) unspecified number. If clone numbering is not needed
|
|
|
470 then the two argument clone_function_name should be used instead.
|
|
|
471 Should not be called directly except for by
|
|
|
472 lto-partition.c:privatize_symbol_name_1. */
|
|
111
|
473
|
|
|
474 tree
|
|
145
|
475 clone_function_name_numbered (const char *name, const char *suffix)
|
|
|
476 {
|
|
|
477 /* Initialize the function->counter mapping the first time it's
|
|
|
478 needed. */
|
|
|
479 if (!clone_fn_ids)
|
|
|
480 clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (64);
|
|
|
481 unsigned int &suffix_counter = clone_fn_ids->get_or_insert (
|
|
|
482 IDENTIFIER_POINTER (get_identifier (name)));
|
|
|
483 return clone_function_name (name, suffix, suffix_counter++);
|
|
|
484 }
|
|
|
485
|
|
|
486 /* Return a new assembler name for a clone of DECL. Apart from string
|
|
|
487 SUFFIX, the new name will end with a unique (for each DECL
|
|
|
488 assembler name) unspecified number. If clone numbering is not
|
|
|
489 needed then the two argument clone_function_name should be used
|
|
|
490 instead. */
|
|
|
491
|
|
|
492 tree
|
|
|
493 clone_function_name_numbered (tree decl, const char *suffix)
|
|
|
494 {
|
|
|
495 tree name = DECL_ASSEMBLER_NAME (decl);
|
|
|
496 return clone_function_name_numbered (IDENTIFIER_POINTER (name),
|
|
|
497 suffix);
|
|
|
498 }
|
|
|
499
|
|
|
500 /* Return a new assembler name for a clone of decl named NAME. Apart
|
|
|
501 from the string SUFFIX, the new name will end with the specified
|
|
|
502 NUMBER. If clone numbering is not needed then the two argument
|
|
|
503 clone_function_name should be used instead. */
|
|
|
504
|
|
|
505 tree
|
|
|
506 clone_function_name (const char *name, const char *suffix,
|
|
|
507 unsigned long number)
|
|
111
|
508 {
|
|
|
509 size_t len = strlen (name);
|
|
|
510 char *tmp_name, *prefix;
|
|
|
511
|
|
|
512 prefix = XALLOCAVEC (char, len + strlen (suffix) + 2);
|
|
|
513 memcpy (prefix, name, len);
|
|
|
514 strcpy (prefix + len + 1, suffix);
|
|
|
515 prefix[len] = symbol_table::symbol_suffix_separator ();
|
|
145
|
516 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number);
|
|
111
|
517 return get_identifier (tmp_name);
|
|
|
518 }
|
|
|
519
|
|
145
|
520 /* Return a new assembler name for a clone of DECL. Apart from the
|
|
|
521 string SUFFIX, the new name will end with the specified NUMBER. If
|
|
|
522 clone numbering is not needed then the two argument
|
|
|
523 clone_function_name should be used instead. */
|
|
|
524
|
|
|
525 tree
|
|
|
526 clone_function_name (tree decl, const char *suffix,
|
|
|
527 unsigned long number)
|
|
|
528 {
|
|
|
529 return clone_function_name (
|
|
|
530 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number);
|
|
|
531 }
|
|
|
532
|
|
|
533 /* Return a new assembler name ending with the string SUFFIX for a
|
|
|
534 clone of DECL. */
|
|
111
|
535
|
|
|
536 tree
|
|
|
537 clone_function_name (tree decl, const char *suffix)
|
|
|
538 {
|
|
145
|
539 tree identifier = DECL_ASSEMBLER_NAME (decl);
|
|
|
540 /* For consistency this needs to behave the same way as
|
|
|
541 ASM_FORMAT_PRIVATE_NAME does, but without the final number
|
|
|
542 suffix. */
|
|
|
543 char *separator = XALLOCAVEC (char, 2);
|
|
|
544 separator[0] = symbol_table::symbol_suffix_separator ();
|
|
|
545 separator[1] = 0;
|
|
|
546 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
|
|
|
547 const char *prefix = "__";
|
|
|
548 #else
|
|
|
549 const char *prefix = "";
|
|
|
550 #endif
|
|
|
551 char *result = ACONCAT ((prefix,
|
|
|
552 IDENTIFIER_POINTER (identifier),
|
|
|
553 separator,
|
|
|
554 suffix,
|
|
|
555 (char*)0));
|
|
|
556 return get_identifier (result);
|
|
111
|
557 }
|
|
|
558
|
|
|
559
|
|
145
|
560 /* Create callgraph node clone with new declaration. The actual body will be
|
|
|
561 copied later at compilation stage. The name of the new clone will be
|
|
|
562 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
|
|
111
|
563
|
|
|
564 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
|
|
|
565 bitmap interface.
|
|
|
566 */
|
|
|
567 cgraph_node *
|
|
|
568 cgraph_node::create_virtual_clone (vec<cgraph_edge *> redirect_callers,
|
|
|
569 vec<ipa_replace_map *, va_gc> *tree_map,
|
|
145
|
570 ipa_param_adjustments *param_adjustments,
|
|
|
571 const char * suffix, unsigned num_suffix)
|
|
111
|
572 {
|
|
|
573 tree old_decl = decl;
|
|
|
574 cgraph_node *new_node = NULL;
|
|
|
575 tree new_decl;
|
|
|
576 size_t len, i;
|
|
|
577 ipa_replace_map *map;
|
|
|
578 char *name;
|
|
|
579
|
|
145
|
580 gcc_checking_assert (versionable);
|
|
|
581 /* TODO: It would be nice if we could recognize that param_adjustments do not
|
|
|
582 actually perform any changes, but at the moment let's require it simply
|
|
|
583 does not exist. */
|
|
|
584 gcc_assert (can_change_signature || !param_adjustments);
|
|
111
|
585
|
|
|
586 /* Make a new FUNCTION_DECL tree node */
|
|
145
|
587 if (!param_adjustments)
|
|
111
|
588 new_decl = copy_node (old_decl);
|
|
|
589 else
|
|
145
|
590 new_decl = param_adjustments->adjust_decl (old_decl);
|
|
111
|
591
|
|
|
592 /* These pointers represent function body and will be populated only when clone
|
|
|
593 is materialized. */
|
|
|
594 gcc_assert (new_decl != old_decl);
|
|
|
595 DECL_STRUCT_FUNCTION (new_decl) = NULL;
|
|
|
596 DECL_ARGUMENTS (new_decl) = NULL;
|
|
|
597 DECL_INITIAL (new_decl) = NULL;
|
|
|
598 DECL_RESULT (new_decl) = NULL;
|
|
145
|
599 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
|
|
111
|
600 sometimes storing only clone decl instead of original. */
|
|
|
601
|
|
|
602 /* Generate a new name for the new version. */
|
|
|
603 len = IDENTIFIER_LENGTH (DECL_NAME (old_decl));
|
|
|
604 name = XALLOCAVEC (char, len + strlen (suffix) + 2);
|
|
|
605 memcpy (name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), len);
|
|
|
606 strcpy (name + len + 1, suffix);
|
|
|
607 name[len] = '.';
|
|
|
608 DECL_NAME (new_decl) = get_identifier (name);
|
|
145
|
609 SET_DECL_ASSEMBLER_NAME (new_decl,
|
|
|
610 clone_function_name (old_decl, suffix, num_suffix));
|
|
111
|
611 SET_DECL_RTL (new_decl, NULL);
|
|
|
612
|
|
131
|
613 new_node = create_clone (new_decl, count, false,
|
|
145
|
614 redirect_callers, false, NULL, param_adjustments,
|
|
|
615 suffix);
|
|
111
|
616
|
|
|
617 /* Update the properties.
|
|
|
618 Make clone visible only within this translation unit. Make sure
|
|
|
619 that is not weak also.
|
|
|
620 ??? We cannot use COMDAT linkage because there is no
|
|
|
621 ABI support for this. */
|
|
|
622 set_new_clone_decl_and_node_flags (new_node);
|
|
145
|
623 new_node->ipcp_clone = ipcp_clone;
|
|
111
|
624 new_node->clone.tree_map = tree_map;
|
|
|
625 if (!implicit_section)
|
|
|
626 new_node->set_section (get_section ());
|
|
|
627
|
|
|
628 /* Clones of global symbols or symbols with unique names are unique. */
|
|
|
629 if ((TREE_PUBLIC (old_decl)
|
|
|
630 && !DECL_EXTERNAL (old_decl)
|
|
|
631 && !DECL_WEAK (old_decl)
|
|
|
632 && !DECL_COMDAT (old_decl))
|
|
|
633 || in_lto_p)
|
|
|
634 new_node->unique_name = true;
|
|
|
635 FOR_EACH_VEC_SAFE_ELT (tree_map, i, map)
|
|
|
636 new_node->maybe_create_reference (map->new_tree, NULL);
|
|
|
637
|
|
|
638 if (ipa_transforms_to_apply.exists ())
|
|
|
639 new_node->ipa_transforms_to_apply
|
|
|
640 = ipa_transforms_to_apply.copy ();
|
|
|
641
|
|
|
642 symtab->call_cgraph_duplication_hooks (this, new_node);
|
|
|
643
|
|
|
644 return new_node;
|
|
|
645 }
|
|
|
646
|
|
|
647 /* callgraph node being removed from symbol table; see if its entry can be
|
|
|
648 replaced by other inline clone. */
|
|
|
649 cgraph_node *
|
|
|
650 cgraph_node::find_replacement (void)
|
|
|
651 {
|
|
|
652 cgraph_node *next_inline_clone, *replacement;
|
|
|
653
|
|
|
654 for (next_inline_clone = clones;
|
|
|
655 next_inline_clone
|
|
|
656 && next_inline_clone->decl != decl;
|
|
|
657 next_inline_clone = next_inline_clone->next_sibling_clone)
|
|
|
658 ;
|
|
|
659
|
|
|
660 /* If there is inline clone of the node being removed, we need
|
|
|
661 to put it into the position of removed node and reorganize all
|
|
|
662 other clones to be based on it. */
|
|
|
663 if (next_inline_clone)
|
|
|
664 {
|
|
|
665 cgraph_node *n;
|
|
|
666 cgraph_node *new_clones;
|
|
|
667
|
|
|
668 replacement = next_inline_clone;
|
|
|
669
|
|
|
670 /* Unlink inline clone from the list of clones of removed node. */
|
|
|
671 if (next_inline_clone->next_sibling_clone)
|
|
|
672 next_inline_clone->next_sibling_clone->prev_sibling_clone
|
|
|
673 = next_inline_clone->prev_sibling_clone;
|
|
|
674 if (next_inline_clone->prev_sibling_clone)
|
|
|
675 {
|
|
|
676 gcc_assert (clones != next_inline_clone);
|
|
|
677 next_inline_clone->prev_sibling_clone->next_sibling_clone
|
|
|
678 = next_inline_clone->next_sibling_clone;
|
|
|
679 }
|
|
|
680 else
|
|
|
681 {
|
|
|
682 gcc_assert (clones == next_inline_clone);
|
|
|
683 clones = next_inline_clone->next_sibling_clone;
|
|
|
684 }
|
|
|
685
|
|
|
686 new_clones = clones;
|
|
|
687 clones = NULL;
|
|
|
688
|
|
|
689 /* Copy clone info. */
|
|
|
690 next_inline_clone->clone = clone;
|
|
|
691
|
|
|
692 /* Now place it into clone tree at same level at NODE. */
|
|
|
693 next_inline_clone->clone_of = clone_of;
|
|
|
694 next_inline_clone->prev_sibling_clone = NULL;
|
|
|
695 next_inline_clone->next_sibling_clone = NULL;
|
|
|
696 if (clone_of)
|
|
|
697 {
|
|
|
698 if (clone_of->clones)
|
|
|
699 clone_of->clones->prev_sibling_clone = next_inline_clone;
|
|
|
700 next_inline_clone->next_sibling_clone = clone_of->clones;
|
|
|
701 clone_of->clones = next_inline_clone;
|
|
|
702 }
|
|
|
703
|
|
|
704 /* Merge the clone list. */
|
|
|
705 if (new_clones)
|
|
|
706 {
|
|
|
707 if (!next_inline_clone->clones)
|
|
|
708 next_inline_clone->clones = new_clones;
|
|
|
709 else
|
|
|
710 {
|
|
|
711 n = next_inline_clone->clones;
|
|
|
712 while (n->next_sibling_clone)
|
|
|
713 n = n->next_sibling_clone;
|
|
|
714 n->next_sibling_clone = new_clones;
|
|
|
715 new_clones->prev_sibling_clone = n;
|
|
|
716 }
|
|
|
717 }
|
|
|
718
|
|
|
719 /* Update clone_of pointers. */
|
|
|
720 n = new_clones;
|
|
|
721 while (n)
|
|
|
722 {
|
|
|
723 n->clone_of = next_inline_clone;
|
|
|
724 n = n->next_sibling_clone;
|
|
|
725 }
|
|
145
|
726
|
|
|
727 /* Update order in order to be able to find a LTO section
|
|
|
728 with function body. */
|
|
|
729 replacement->order = order;
|
|
|
730
|
|
111
|
731 return replacement;
|
|
|
732 }
|
|
|
733 else
|
|
|
734 return NULL;
|
|
|
735 }
|
|
|
736
|
|
|
737 /* Like cgraph_set_call_stmt but walk the clone tree and update all
|
|
|
738 clones sharing the same function body.
|
|
|
739 When WHOLE_SPECULATIVE_EDGES is true, all three components of
|
|
|
740 speculative edge gets updated. Otherwise we update only direct
|
|
|
741 call. */
|
|
|
742
|
|
|
743 void
|
|
|
744 cgraph_node::set_call_stmt_including_clones (gimple *old_stmt,
|
|
|
745 gcall *new_stmt,
|
|
|
746 bool update_speculative)
|
|
|
747 {
|
|
|
748 cgraph_node *node;
|
|
145
|
749 cgraph_edge *master_edge = get_edge (old_stmt);
|
|
111
|
750
|
|
145
|
751 if (master_edge)
|
|
|
752 cgraph_edge::set_call_stmt (master_edge, new_stmt, update_speculative);
|
|
111
|
753
|
|
|
754 node = clones;
|
|
|
755 if (node)
|
|
|
756 while (node != this)
|
|
|
757 {
|
|
|
758 cgraph_edge *edge = node->get_edge (old_stmt);
|
|
|
759 if (edge)
|
|
|
760 {
|
|
145
|
761 edge = cgraph_edge::set_call_stmt (edge, new_stmt,
|
|
|
762 update_speculative);
|
|
111
|
763 /* If UPDATE_SPECULATIVE is false, it means that we are turning
|
|
|
764 speculative call into a real code sequence. Update the
|
|
|
765 callgraph edges. */
|
|
|
766 if (edge->speculative && !update_speculative)
|
|
|
767 {
|
|
145
|
768 cgraph_edge *indirect = edge->speculative_call_indirect_edge ();
|
|
111
|
769
|
|
145
|
770 for (cgraph_edge *next, *direct
|
|
|
771 = edge->first_speculative_call_target ();
|
|
|
772 direct;
|
|
|
773 direct = next)
|
|
|
774 {
|
|
|
775 next = direct->next_speculative_call_target ();
|
|
|
776 direct->speculative_call_target_ref ()->speculative = false;
|
|
|
777 direct->speculative = false;
|
|
|
778 }
|
|
111
|
779 indirect->speculative = false;
|
|
|
780 }
|
|
|
781 }
|
|
|
782 if (node->clones)
|
|
|
783 node = node->clones;
|
|
|
784 else if (node->next_sibling_clone)
|
|
|
785 node = node->next_sibling_clone;
|
|
|
786 else
|
|
|
787 {
|
|
|
788 while (node != this && !node->next_sibling_clone)
|
|
|
789 node = node->clone_of;
|
|
|
790 if (node != this)
|
|
|
791 node = node->next_sibling_clone;
|
|
|
792 }
|
|
|
793 }
|
|
|
794 }
|
|
|
795
|
|
|
796 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
|
|
|
797 same function body. If clones already have edge for OLD_STMT; only
|
|
|
798 update the edge same way as cgraph_set_call_stmt_including_clones does.
|
|
|
799
|
|
|
800 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
|
|
|
801 frequencies of the clones. */
|
|
|
802
|
|
|
803 void
|
|
|
804 cgraph_node::create_edge_including_clones (cgraph_node *callee,
|
|
|
805 gimple *old_stmt, gcall *stmt,
|
|
|
806 profile_count count,
|
|
|
807 cgraph_inline_failed_t reason)
|
|
|
808 {
|
|
|
809 cgraph_node *node;
|
|
|
810
|
|
|
811 if (!get_edge (stmt))
|
|
|
812 {
|
|
145
|
813 cgraph_edge *edge = create_edge (callee, stmt, count);
|
|
111
|
814 edge->inline_failed = reason;
|
|
|
815 }
|
|
|
816
|
|
|
817 node = clones;
|
|
|
818 if (node)
|
|
|
819 while (node != this)
|
|
|
820 /* Thunk clones do not get updated while copying inline function body. */
|
|
|
821 if (!node->thunk.thunk_p)
|
|
|
822 {
|
|
|
823 cgraph_edge *edge = node->get_edge (old_stmt);
|
|
|
824
|
|
|
825 /* It is possible that clones already contain the edge while
|
|
|
826 master didn't. Either we promoted indirect call into direct
|
|
|
827 call in the clone or we are processing clones of unreachable
|
|
|
828 master where edges has been removed. */
|
|
|
829 if (edge)
|
|
145
|
830 edge = cgraph_edge::set_call_stmt (edge, stmt);
|
|
111
|
831 else if (! node->get_edge (stmt))
|
|
|
832 {
|
|
131
|
833 edge = node->create_edge (callee, stmt, count);
|
|
111
|
834 edge->inline_failed = reason;
|
|
|
835 }
|
|
|
836
|
|
|
837 if (node->clones)
|
|
|
838 node = node->clones;
|
|
|
839 else if (node->next_sibling_clone)
|
|
|
840 node = node->next_sibling_clone;
|
|
|
841 else
|
|
|
842 {
|
|
|
843 while (node != this && !node->next_sibling_clone)
|
|
|
844 node = node->clone_of;
|
|
|
845 if (node != this)
|
|
|
846 node = node->next_sibling_clone;
|
|
|
847 }
|
|
|
848 }
|
|
|
849 }
|
|
|
850
|
|
|
851 /* Remove the node from cgraph and all inline clones inlined into it.
|
|
|
852 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
|
|
|
853 removed. This allows to call the function from outer loop walking clone
|
|
|
854 tree. */
|
|
|
855
|
|
|
856 bool
|
|
|
857 cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node)
|
|
|
858 {
|
|
|
859 cgraph_edge *e, *next;
|
|
|
860 bool found = false;
|
|
|
861
|
|
|
862 if (this == forbidden_node)
|
|
|
863 {
|
|
145
|
864 cgraph_edge::remove (callers);
|
|
111
|
865 return true;
|
|
|
866 }
|
|
|
867 for (e = callees; e; e = next)
|
|
|
868 {
|
|
|
869 next = e->next_callee;
|
|
|
870 if (!e->inline_failed)
|
|
|
871 found |= e->callee->remove_symbol_and_inline_clones (forbidden_node);
|
|
|
872 }
|
|
|
873 remove ();
|
|
|
874 return found;
|
|
|
875 }
|
|
|
876
|
|
|
877 /* The edges representing the callers of the NEW_VERSION node were
|
|
|
878 fixed by cgraph_function_versioning (), now the call_expr in their
|
|
|
879 respective tree code should be updated to call the NEW_VERSION. */
|
|
|
880
|
|
|
881 static void
|
|
|
882 update_call_expr (cgraph_node *new_version)
|
|
|
883 {
|
|
|
884 cgraph_edge *e;
|
|
|
885
|
|
|
886 gcc_assert (new_version);
|
|
|
887
|
|
|
888 /* Update the call expr on the edges to call the new version. */
|
|
|
889 for (e = new_version->callers; e; e = e->next_caller)
|
|
|
890 {
|
|
|
891 function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl);
|
|
|
892 gimple_call_set_fndecl (e->call_stmt, new_version->decl);
|
|
|
893 maybe_clean_eh_stmt_fn (inner_function, e->call_stmt);
|
|
|
894 }
|
|
|
895 }
|
|
|
896
|
|
|
897
|
|
|
898 /* Create a new cgraph node which is the new version of
|
|
|
899 callgraph node. REDIRECT_CALLERS holds the callers
|
|
|
900 edges which should be redirected to point to
|
|
|
901 NEW_VERSION. ALL the callees edges of the node
|
|
|
902 are cloned to the new version node. Return the new
|
|
|
903 version node.
|
|
|
904
|
|
|
905 If non-NULL BLOCK_TO_COPY determine what basic blocks
|
|
|
906 was copied to prevent duplications of calls that are dead
|
|
|
907 in the clone. */
|
|
|
908
|
|
|
909 cgraph_node *
|
|
|
910 cgraph_node::create_version_clone (tree new_decl,
|
|
|
911 vec<cgraph_edge *> redirect_callers,
|
|
|
912 bitmap bbs_to_copy,
|
|
|
913 const char *suffix)
|
|
|
914 {
|
|
|
915 cgraph_node *new_version;
|
|
|
916 cgraph_edge *e;
|
|
|
917 unsigned i;
|
|
|
918
|
|
|
919 new_version = cgraph_node::create (new_decl);
|
|
|
920
|
|
|
921 new_version->analyzed = analyzed;
|
|
|
922 new_version->definition = definition;
|
|
|
923 new_version->local = local;
|
|
|
924 new_version->externally_visible = false;
|
|
|
925 new_version->no_reorder = no_reorder;
|
|
145
|
926 new_version->local = new_version->definition;
|
|
|
927 new_version->inlined_to = inlined_to;
|
|
111
|
928 new_version->rtl = rtl;
|
|
|
929 new_version->count = count;
|
|
145
|
930 new_version->unit_id = unit_id;
|
|
|
931 new_version->merged_comdat = merged_comdat;
|
|
|
932 new_version->merged_extern_inline = merged_extern_inline;
|
|
111
|
933
|
|
|
934 for (e = callees; e; e=e->next_callee)
|
|
|
935 if (!bbs_to_copy
|
|
|
936 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
|
|
|
937 e->clone (new_version, e->call_stmt,
|
|
|
938 e->lto_stmt_uid, count, count,
|
|
|
939 true);
|
|
|
940 for (e = indirect_calls; e; e=e->next_callee)
|
|
|
941 if (!bbs_to_copy
|
|
|
942 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
|
|
|
943 e->clone (new_version, e->call_stmt,
|
|
|
944 e->lto_stmt_uid, count, count,
|
|
|
945 true);
|
|
|
946 FOR_EACH_VEC_ELT (redirect_callers, i, e)
|
|
|
947 {
|
|
|
948 /* Redirect calls to the old version node to point to its new
|
|
|
949 version. */
|
|
|
950 e->redirect_callee (new_version);
|
|
|
951 }
|
|
|
952
|
|
|
953 dump_callgraph_transformation (this, new_version, suffix);
|
|
|
954
|
|
|
955 return new_version;
|
|
|
956 }
|
|
|
957
|
|
|
958 /* Perform function versioning.
|
|
|
959 Function versioning includes copying of the tree and
|
|
|
960 a callgraph update (creating a new cgraph node and updating
|
|
|
961 its callees and callers).
|
|
|
962
|
|
|
963 REDIRECT_CALLERS varray includes the edges to be redirected
|
|
|
964 to the new version.
|
|
|
965
|
|
|
966 TREE_MAP is a mapping of tree nodes we want to replace with
|
|
|
967 new ones (according to results of prior analysis).
|
|
|
968
|
|
|
969 If non-NULL ARGS_TO_SKIP determine function parameters to remove
|
|
|
970 from new version.
|
|
|
971 If SKIP_RETURN is true, the new version will return void.
|
|
|
972 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
|
|
|
973 If non_NULL NEW_ENTRY determine new entry BB of the clone.
|
|
|
974
|
|
145
|
975 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
|
|
|
976 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
|
|
|
977 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
|
|
|
978 of the declaration.
|
|
|
979
|
|
111
|
980 Return the new version's cgraph node. */
|
|
|
981
|
|
|
982 cgraph_node *
|
|
|
983 cgraph_node::create_version_clone_with_body
|
|
|
984 (vec<cgraph_edge *> redirect_callers,
|
|
145
|
985 vec<ipa_replace_map *, va_gc> *tree_map,
|
|
|
986 ipa_param_adjustments *param_adjustments,
|
|
|
987 bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix,
|
|
|
988 tree target_attributes)
|
|
111
|
989 {
|
|
|
990 tree old_decl = decl;
|
|
|
991 cgraph_node *new_version_node = NULL;
|
|
|
992 tree new_decl;
|
|
|
993
|
|
|
994 if (!tree_versionable_function_p (old_decl))
|
|
|
995 return NULL;
|
|
|
996
|
|
145
|
997 /* TODO: Restore an assert that we do not change signature if
|
|
|
998 can_change_signature is false. We cannot just check that
|
|
|
999 param_adjustments is NULL because unfortunately ipa-split removes return
|
|
|
1000 values from such functions. */
|
|
111
|
1001
|
|
|
1002 /* Make a new FUNCTION_DECL tree node for the new version. */
|
|
145
|
1003 if (param_adjustments)
|
|
|
1004 new_decl = param_adjustments->adjust_decl (old_decl);
|
|
111
|
1005 else
|
|
145
|
1006 new_decl = copy_node (old_decl);
|
|
111
|
1007
|
|
|
1008 /* Generate a new name for the new version. */
|
|
145
|
1009 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl, suffix);
|
|
111
|
1010 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
|
|
|
1011 SET_DECL_RTL (new_decl, NULL);
|
|
|
1012
|
|
131
|
1013 DECL_VIRTUAL_P (new_decl) = 0;
|
|
|
1014
|
|
145
|
1015 if (target_attributes)
|
|
|
1016 {
|
|
|
1017 DECL_ATTRIBUTES (new_decl) = target_attributes;
|
|
|
1018
|
|
|
1019 location_t saved_loc = input_location;
|
|
|
1020 tree v = TREE_VALUE (target_attributes);
|
|
|
1021 input_location = DECL_SOURCE_LOCATION (new_decl);
|
|
|
1022 bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1);
|
|
|
1023 input_location = saved_loc;
|
|
|
1024 if (!r)
|
|
|
1025 return NULL;
|
|
|
1026 }
|
|
|
1027
|
|
111
|
1028 /* When the old decl was a con-/destructor make sure the clone isn't. */
|
|
|
1029 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
|
|
|
1030 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
|
|
145
|
1031 DECL_SET_IS_OPERATOR_NEW (new_decl, 0);
|
|
|
1032 DECL_SET_IS_OPERATOR_DELETE (new_decl, 0);
|
|
111
|
1033
|
|
|
1034 /* Create the new version's call-graph node.
|
|
|
1035 and update the edges of the new node. */
|
|
|
1036 new_version_node = create_version_clone (new_decl, redirect_callers,
|
|
|
1037 bbs_to_copy, suffix);
|
|
|
1038
|
|
|
1039 if (ipa_transforms_to_apply.exists ())
|
|
|
1040 new_version_node->ipa_transforms_to_apply
|
|
|
1041 = ipa_transforms_to_apply.copy ();
|
|
|
1042 /* Copy the OLD_VERSION_NODE function tree to the new version. */
|
|
145
|
1043 tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments,
|
|
|
1044 false, bbs_to_copy, new_entry_block);
|
|
111
|
1045
|
|
|
1046 /* Update the new version's properties.
|
|
|
1047 Make The new version visible only within this translation unit. Make sure
|
|
|
1048 that is not weak also.
|
|
|
1049 ??? We cannot use COMDAT linkage because there is no
|
|
|
1050 ABI support for this. */
|
|
|
1051 new_version_node->make_decl_local ();
|
|
|
1052 DECL_VIRTUAL_P (new_version_node->decl) = 0;
|
|
|
1053 new_version_node->externally_visible = 0;
|
|
145
|
1054 new_version_node->local = 1;
|
|
111
|
1055 new_version_node->lowered = true;
|
|
|
1056 if (!implicit_section)
|
|
|
1057 new_version_node->set_section (get_section ());
|
|
|
1058 /* Clones of global symbols or symbols with unique names are unique. */
|
|
|
1059 if ((TREE_PUBLIC (old_decl)
|
|
|
1060 && !DECL_EXTERNAL (old_decl)
|
|
|
1061 && !DECL_WEAK (old_decl)
|
|
|
1062 && !DECL_COMDAT (old_decl))
|
|
|
1063 || in_lto_p)
|
|
|
1064 new_version_node->unique_name = true;
|
|
|
1065
|
|
|
1066 /* Update the call_expr on the edges to call the new version node. */
|
|
|
1067 update_call_expr (new_version_node);
|
|
|
1068
|
|
131
|
1069 symtab->call_cgraph_insertion_hooks (new_version_node);
|
|
111
|
1070 return new_version_node;
|
|
|
1071 }
|
|
|
1072
|
|
145
|
1073 /* Remove the node from the tree of virtual and inline clones and make it a
|
|
|
1074 standalone node - not a clone any more. */
|
|
|
1075
|
|
|
1076 void cgraph_node::remove_from_clone_tree ()
|
|
|
1077 {
|
|
|
1078 if (next_sibling_clone)
|
|
|
1079 next_sibling_clone->prev_sibling_clone = prev_sibling_clone;
|
|
|
1080 if (prev_sibling_clone)
|
|
|
1081 prev_sibling_clone->next_sibling_clone = next_sibling_clone;
|
|
|
1082 else
|
|
|
1083 clone_of->clones = next_sibling_clone;
|
|
|
1084 next_sibling_clone = NULL;
|
|
|
1085 prev_sibling_clone = NULL;
|
|
|
1086 clone_of = NULL;
|
|
|
1087 }
|
|
|
1088
|
|
111
|
1089 /* Given virtual clone, turn it into actual clone. */
|
|
|
1090
|
|
|
1091 static void
|
|
|
1092 cgraph_materialize_clone (cgraph_node *node)
|
|
|
1093 {
|
|
|
1094 bitmap_obstack_initialize (NULL);
|
|
|
1095 node->former_clone_of = node->clone_of->decl;
|
|
|
1096 if (node->clone_of->former_clone_of)
|
|
|
1097 node->former_clone_of = node->clone_of->former_clone_of;
|
|
|
1098 /* Copy the OLD_VERSION_NODE function tree to the new version. */
|
|
|
1099 tree_function_versioning (node->clone_of->decl, node->decl,
|
|
145
|
1100 node->clone.tree_map, node->clone.param_adjustments,
|
|
|
1101 true, NULL, NULL);
|
|
111
|
1102 if (symtab->dump_file)
|
|
|
1103 {
|
|
|
1104 dump_function_to_file (node->clone_of->decl, symtab->dump_file,
|
|
|
1105 dump_flags);
|
|
|
1106 dump_function_to_file (node->decl, symtab->dump_file, dump_flags);
|
|
|
1107 }
|
|
|
1108
|
|
145
|
1109 cgraph_node *clone_of = node->clone_of;
|
|
111
|
1110 /* Function is no longer clone. */
|
|
145
|
1111 node->remove_from_clone_tree ();
|
|
|
1112 if (!clone_of->analyzed && !clone_of->clones)
|
|
111
|
1113 {
|
|
145
|
1114 clone_of->release_body ();
|
|
|
1115 clone_of->remove_callees ();
|
|
|
1116 clone_of->remove_all_references ();
|
|
111
|
1117 }
|
|
|
1118 bitmap_obstack_release (NULL);
|
|
|
1119 }
|
|
|
1120
|
|
|
1121 /* Once all functions from compilation unit are in memory, produce all clones
|
|
|
1122 and update all calls. We might also do this on demand if we don't want to
|
|
|
1123 bring all functions to memory prior compilation, but current WHOPR
|
|
|
1124 implementation does that and it is a bit easier to keep everything right in
|
|
|
1125 this order. */
|
|
|
1126
|
|
|
1127 void
|
|
|
1128 symbol_table::materialize_all_clones (void)
|
|
|
1129 {
|
|
|
1130 cgraph_node *node;
|
|
|
1131 bool stabilized = false;
|
|
|
1132
|
|
|
1133
|
|
|
1134 if (symtab->dump_file)
|
|
|
1135 fprintf (symtab->dump_file, "Materializing clones\n");
|
|
|
1136
|
|
|
1137 cgraph_node::checking_verify_cgraph_nodes ();
|
|
|
1138
|
|
|
1139 /* We can also do topological order, but number of iterations should be
|
|
|
1140 bounded by number of IPA passes since single IPA pass is probably not
|
|
|
1141 going to create clones of clones it created itself. */
|
|
|
1142 while (!stabilized)
|
|
|
1143 {
|
|
|
1144 stabilized = true;
|
|
|
1145 FOR_EACH_FUNCTION (node)
|
|
|
1146 {
|
|
|
1147 if (node->clone_of && node->decl != node->clone_of->decl
|
|
|
1148 && !gimple_has_body_p (node->decl))
|
|
|
1149 {
|
|
|
1150 if (!node->clone_of->clone_of)
|
|
|
1151 node->clone_of->get_untransformed_body ();
|
|
|
1152 if (gimple_has_body_p (node->clone_of->decl))
|
|
|
1153 {
|
|
|
1154 if (symtab->dump_file)
|
|
|
1155 {
|
|
|
1156 fprintf (symtab->dump_file, "cloning %s to %s\n",
|
|
145
|
1157 node->clone_of->dump_name (),
|
|
|
1158 node->dump_name ());
|
|
111
|
1159 if (node->clone.tree_map)
|
|
|
1160 {
|
|
|
1161 unsigned int i;
|
|
|
1162 fprintf (symtab->dump_file, " replace map: ");
|
|
|
1163 for (i = 0;
|
|
|
1164 i < vec_safe_length (node->clone.tree_map);
|
|
|
1165 i++)
|
|
|
1166 {
|
|
|
1167 ipa_replace_map *replace_info;
|
|
|
1168 replace_info = (*node->clone.tree_map)[i];
|
|
145
|
1169 fprintf (symtab->dump_file, "%i -> ",
|
|
|
1170 (*node->clone.tree_map)[i]->parm_num);
|
|
111
|
1171 print_generic_expr (symtab->dump_file,
|
|
|
1172 replace_info->new_tree);
|
|
|
1173 }
|
|
|
1174 fprintf (symtab->dump_file, "\n");
|
|
|
1175 }
|
|
145
|
1176 if (node->clone.param_adjustments)
|
|
|
1177 node->clone.param_adjustments->dump (symtab->dump_file);
|
|
111
|
1178 }
|
|
|
1179 cgraph_materialize_clone (node);
|
|
|
1180 stabilized = false;
|
|
|
1181 }
|
|
|
1182 }
|
|
|
1183 }
|
|
|
1184 }
|
|
|
1185 FOR_EACH_FUNCTION (node)
|
|
|
1186 if (!node->analyzed && node->callees)
|
|
|
1187 {
|
|
|
1188 node->remove_callees ();
|
|
|
1189 node->remove_all_references ();
|
|
|
1190 }
|
|
|
1191 else
|
|
|
1192 node->clear_stmts_in_references ();
|
|
|
1193 if (symtab->dump_file)
|
|
|
1194 fprintf (symtab->dump_file, "Materialization Call site updates done.\n");
|
|
|
1195
|
|
|
1196 cgraph_node::checking_verify_cgraph_nodes ();
|
|
|
1197
|
|
|
1198 symtab->remove_unreachable_nodes (symtab->dump_file);
|
|
|
1199 }
|
|
|
1200
|
|
|
1201 #include "gt-cgraphclones.h"
|
