|
0
|
1 /* The tracer pass for the GNU compiler.
|
|
|
2 Contributed by Jan Hubicka, SuSE Labs.
|
|
|
3 Adapted to work on GIMPLE instead of RTL by Robert Kidd, UIUC.
|
|
111
|
4 Copyright (C) 2001-2017 Free Software Foundation, Inc.
|
|
0
|
5
|
|
|
6 This file is part of GCC.
|
|
|
7
|
|
|
8 GCC is free software; you can redistribute it and/or modify it
|
|
|
9 under the terms of the GNU General Public License as published by
|
|
|
10 the Free Software Foundation; either version 3, or (at your option)
|
|
|
11 any later version.
|
|
|
12
|
|
|
13 GCC is distributed in the hope that it will be useful, but WITHOUT
|
|
|
14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
|
|
|
15 or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
|
|
|
16 License for more details.
|
|
|
17
|
|
|
18 You should have received a copy of the GNU General Public License
|
|
|
19 along with GCC; see the file COPYING3. If not see
|
|
|
20 <http://www.gnu.org/licenses/>. */
|
|
|
21
|
|
|
22 /* This pass performs the tail duplication needed for superblock formation.
|
|
|
23 For more information see:
|
|
|
24
|
|
|
25 Design and Analysis of Profile-Based Optimization in Compaq's
|
|
|
26 Compilation Tools for Alpha; Journal of Instruction-Level
|
|
|
27 Parallelism 3 (2000) 1-25
|
|
|
28
|
|
|
29 Unlike Compaq's implementation we don't do the loop peeling as most
|
|
|
30 probably a better job can be done by a special pass and we don't
|
|
|
31 need to worry too much about the code size implications as the tail
|
|
|
32 duplicates are crossjumped again if optimizations are not
|
|
|
33 performed. */
|
|
|
34
|
|
|
35
|
|
|
36 #include "config.h"
|
|
|
37 #include "system.h"
|
|
|
38 #include "coretypes.h"
|
|
111
|
39 #include "backend.h"
|
|
0
|
40 #include "rtl.h"
|
|
111
|
41 #include "tree.h"
|
|
|
42 #include "gimple.h"
|
|
|
43 #include "cfghooks.h"
|
|
|
44 #include "tree-pass.h"
|
|
|
45 #include "profile.h"
|
|
|
46 #include "cfganal.h"
|
|
0
|
47 #include "params.h"
|
|
111
|
48 #include "gimple-iterator.h"
|
|
|
49 #include "tree-cfg.h"
|
|
|
50 #include "tree-ssa.h"
|
|
0
|
51 #include "tree-inline.h"
|
|
111
|
52 #include "cfgloop.h"
|
|
|
53 #include "fibonacci_heap.h"
|
|
|
54 #include "tracer.h"
|
|
0
|
55
|
|
|
56 static int count_insns (basic_block);
|
|
|
57 static bool better_p (const_edge, const_edge);
|
|
|
58 static edge find_best_successor (basic_block);
|
|
|
59 static edge find_best_predecessor (basic_block);
|
|
|
60 static int find_trace (basic_block, basic_block *);
|
|
|
61
|
|
|
62 /* Minimal outgoing edge probability considered for superblock formation. */
|
|
|
63 static int probability_cutoff;
|
|
|
64 static int branch_ratio_cutoff;
|
|
|
65
|
|
|
66 /* A bit BB->index is set if BB has already been seen, i.e. it is
|
|
|
67 connected to some trace already. */
|
|
111
|
68 static sbitmap bb_seen;
|
|
0
|
69
|
|
|
70 static inline void
|
|
|
71 mark_bb_seen (basic_block bb)
|
|
|
72 {
|
|
111
|
73 unsigned int size = SBITMAP_SIZE (bb_seen);
|
|
0
|
74
|
|
|
75 if ((unsigned int)bb->index >= size)
|
|
|
76 bb_seen = sbitmap_resize (bb_seen, size * 2, 0);
|
|
|
77
|
|
111
|
78 bitmap_set_bit (bb_seen, bb->index);
|
|
0
|
79 }
|
|
|
80
|
|
|
81 static inline bool
|
|
|
82 bb_seen_p (basic_block bb)
|
|
|
83 {
|
|
111
|
84 return bitmap_bit_p (bb_seen, bb->index);
|
|
0
|
85 }
|
|
|
86
|
|
|
87 /* Return true if we should ignore the basic block for purposes of tracing. */
|
|
111
|
88 bool
|
|
0
|
89 ignore_bb_p (const_basic_block bb)
|
|
|
90 {
|
|
|
91 if (bb->index < NUM_FIXED_BLOCKS)
|
|
|
92 return true;
|
|
|
93 if (optimize_bb_for_size_p (bb))
|
|
|
94 return true;
|
|
111
|
95
|
|
|
96 if (gimple *g = last_stmt (CONST_CAST_BB (bb)))
|
|
|
97 {
|
|
|
98 /* A transaction is a single entry multiple exit region. It
|
|
|
99 must be duplicated in its entirety or not at all. */
|
|
|
100 if (gimple_code (g) == GIMPLE_TRANSACTION)
|
|
|
101 return true;
|
|
|
102
|
|
|
103 /* An IFN_UNIQUE call must be duplicated as part of its group,
|
|
|
104 or not at all. */
|
|
|
105 if (is_gimple_call (g)
|
|
|
106 && gimple_call_internal_p (g)
|
|
|
107 && gimple_call_internal_unique_p (g))
|
|
|
108 return true;
|
|
|
109 }
|
|
|
110
|
|
0
|
111 return false;
|
|
|
112 }
|
|
|
113
|
|
|
114 /* Return number of instructions in the block. */
|
|
|
115
|
|
|
116 static int
|
|
|
117 count_insns (basic_block bb)
|
|
|
118 {
|
|
|
119 gimple_stmt_iterator gsi;
|
|
111
|
120 gimple *stmt;
|
|
0
|
121 int n = 0;
|
|
|
122
|
|
|
123 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
124 {
|
|
|
125 stmt = gsi_stmt (gsi);
|
|
|
126 n += estimate_num_insns (stmt, &eni_size_weights);
|
|
|
127 }
|
|
|
128 return n;
|
|
|
129 }
|
|
|
130
|
|
|
131 /* Return true if E1 is more frequent than E2. */
|
|
|
132 static bool
|
|
|
133 better_p (const_edge e1, const_edge e2)
|
|
|
134 {
|
|
111
|
135 if (e1->count ().initialized_p () && e2->count ().initialized_p ()
|
|
|
136 && ((e1->count () > e2->count ()) || (e1->count () < e2->count ())))
|
|
|
137 return e1->count () > e2->count ();
|
|
|
138 if (EDGE_FREQUENCY (e1) != EDGE_FREQUENCY (e2))
|
|
|
139 return EDGE_FREQUENCY (e1) > EDGE_FREQUENCY (e2);
|
|
0
|
140 /* This is needed to avoid changes in the decision after
|
|
|
141 CFG is modified. */
|
|
|
142 if (e1->src != e2->src)
|
|
|
143 return e1->src->index > e2->src->index;
|
|
|
144 return e1->dest->index > e2->dest->index;
|
|
|
145 }
|
|
|
146
|
|
|
147 /* Return most frequent successor of basic block BB. */
|
|
|
148
|
|
|
149 static edge
|
|
|
150 find_best_successor (basic_block bb)
|
|
|
151 {
|
|
|
152 edge e;
|
|
|
153 edge best = NULL;
|
|
|
154 edge_iterator ei;
|
|
|
155
|
|
|
156 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
157 if (!best || better_p (e, best))
|
|
|
158 best = e;
|
|
|
159 if (!best || ignore_bb_p (best->dest))
|
|
|
160 return NULL;
|
|
111
|
161 if (best->probability.initialized_p ()
|
|
|
162 && best->probability.to_reg_br_prob_base () <= probability_cutoff)
|
|
0
|
163 return NULL;
|
|
|
164 return best;
|
|
|
165 }
|
|
|
166
|
|
|
167 /* Return most frequent predecessor of basic block BB. */
|
|
|
168
|
|
|
169 static edge
|
|
|
170 find_best_predecessor (basic_block bb)
|
|
|
171 {
|
|
|
172 edge e;
|
|
|
173 edge best = NULL;
|
|
|
174 edge_iterator ei;
|
|
|
175
|
|
|
176 FOR_EACH_EDGE (e, ei, bb->preds)
|
|
|
177 if (!best || better_p (e, best))
|
|
|
178 best = e;
|
|
|
179 if (!best || ignore_bb_p (best->src))
|
|
|
180 return NULL;
|
|
|
181 if (EDGE_FREQUENCY (best) * REG_BR_PROB_BASE
|
|
|
182 < bb->frequency * branch_ratio_cutoff)
|
|
|
183 return NULL;
|
|
|
184 return best;
|
|
|
185 }
|
|
|
186
|
|
|
187 /* Find the trace using bb and record it in the TRACE array.
|
|
|
188 Return number of basic blocks recorded. */
|
|
|
189
|
|
|
190 static int
|
|
|
191 find_trace (basic_block bb, basic_block *trace)
|
|
|
192 {
|
|
|
193 int i = 0;
|
|
|
194 edge e;
|
|
|
195
|
|
|
196 if (dump_file)
|
|
|
197 fprintf (dump_file, "Trace seed %i [%i]", bb->index, bb->frequency);
|
|
|
198
|
|
|
199 while ((e = find_best_predecessor (bb)) != NULL)
|
|
|
200 {
|
|
|
201 basic_block bb2 = e->src;
|
|
|
202 if (bb_seen_p (bb2) || (e->flags & (EDGE_DFS_BACK | EDGE_COMPLEX))
|
|
|
203 || find_best_successor (bb2) != e)
|
|
|
204 break;
|
|
|
205 if (dump_file)
|
|
|
206 fprintf (dump_file, ",%i [%i]", bb->index, bb->frequency);
|
|
|
207 bb = bb2;
|
|
|
208 }
|
|
|
209 if (dump_file)
|
|
|
210 fprintf (dump_file, " forward %i [%i]", bb->index, bb->frequency);
|
|
|
211 trace[i++] = bb;
|
|
|
212
|
|
|
213 /* Follow the trace in forward direction. */
|
|
|
214 while ((e = find_best_successor (bb)) != NULL)
|
|
|
215 {
|
|
|
216 bb = e->dest;
|
|
|
217 if (bb_seen_p (bb) || (e->flags & (EDGE_DFS_BACK | EDGE_COMPLEX))
|
|
|
218 || find_best_predecessor (bb) != e)
|
|
|
219 break;
|
|
|
220 if (dump_file)
|
|
|
221 fprintf (dump_file, ",%i [%i]", bb->index, bb->frequency);
|
|
|
222 trace[i++] = bb;
|
|
|
223 }
|
|
|
224 if (dump_file)
|
|
|
225 fprintf (dump_file, "\n");
|
|
|
226 return i;
|
|
|
227 }
|
|
|
228
|
|
111
|
229 /* Duplicate block BB2, placing it after BB in the CFG. Return the
|
|
|
230 newly created block. */
|
|
|
231 basic_block
|
|
|
232 transform_duplicate (basic_block bb, basic_block bb2)
|
|
|
233 {
|
|
|
234 edge e;
|
|
|
235 basic_block copy;
|
|
|
236
|
|
|
237 e = find_edge (bb, bb2);
|
|
|
238
|
|
|
239 copy = duplicate_block (bb2, e, bb);
|
|
|
240 flush_pending_stmts (e);
|
|
|
241
|
|
|
242 add_phi_args_after_copy (©, 1, NULL);
|
|
|
243
|
|
|
244 return (copy);
|
|
|
245 }
|
|
|
246
|
|
0
|
247 /* Look for basic blocks in frequency order, construct traces and tail duplicate
|
|
|
248 if profitable. */
|
|
|
249
|
|
111
|
250 static bool
|
|
0
|
251 tail_duplicate (void)
|
|
|
252 {
|
|
111
|
253 auto_vec<fibonacci_node<long, basic_block_def>*> blocks;
|
|
|
254 blocks.safe_grow_cleared (last_basic_block_for_fn (cfun));
|
|
|
255
|
|
|
256 basic_block *trace = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
|
|
|
257 int *counts = XNEWVEC (int, last_basic_block_for_fn (cfun));
|
|
0
|
258 int ninsns = 0, nduplicated = 0;
|
|
|
259 gcov_type weighted_insns = 0, traced_insns = 0;
|
|
111
|
260 fibonacci_heap<long, basic_block_def> heap (LONG_MIN);
|
|
0
|
261 gcov_type cover_insns;
|
|
|
262 int max_dup_insns;
|
|
|
263 basic_block bb;
|
|
111
|
264 bool changed = false;
|
|
0
|
265
|
|
|
266 /* Create an oversized sbitmap to reduce the chance that we need to
|
|
|
267 resize it. */
|
|
111
|
268 bb_seen = sbitmap_alloc (last_basic_block_for_fn (cfun) * 2);
|
|
|
269 bitmap_clear (bb_seen);
|
|
0
|
270 initialize_original_copy_tables ();
|
|
|
271
|
|
111
|
272 if (profile_info && profile_status_for_fn (cfun) == PROFILE_READ)
|
|
0
|
273 probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK);
|
|
|
274 else
|
|
|
275 probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY);
|
|
|
276 probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff;
|
|
|
277
|
|
|
278 branch_ratio_cutoff =
|
|
|
279 (REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO));
|
|
|
280
|
|
111
|
281 FOR_EACH_BB_FN (bb, cfun)
|
|
0
|
282 {
|
|
|
283 int n = count_insns (bb);
|
|
|
284 if (!ignore_bb_p (bb))
|
|
111
|
285 blocks[bb->index] = heap.insert (-bb->frequency, bb);
|
|
0
|
286
|
|
|
287 counts [bb->index] = n;
|
|
|
288 ninsns += n;
|
|
|
289 weighted_insns += n * bb->frequency;
|
|
|
290 }
|
|
|
291
|
|
111
|
292 if (profile_info && profile_status_for_fn (cfun) == PROFILE_READ)
|
|
0
|
293 cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK);
|
|
|
294 else
|
|
|
295 cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE);
|
|
|
296 cover_insns = (weighted_insns * cover_insns + 50) / 100;
|
|
|
297 max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100;
|
|
|
298
|
|
|
299 while (traced_insns < cover_insns && nduplicated < max_dup_insns
|
|
111
|
300 && !heap.empty ())
|
|
0
|
301 {
|
|
111
|
302 basic_block bb = heap.extract_min ();
|
|
0
|
303 int n, pos;
|
|
|
304
|
|
|
305 if (!bb)
|
|
|
306 break;
|
|
|
307
|
|
|
308 blocks[bb->index] = NULL;
|
|
|
309
|
|
|
310 if (ignore_bb_p (bb))
|
|
|
311 continue;
|
|
|
312 gcc_assert (!bb_seen_p (bb));
|
|
|
313
|
|
|
314 n = find_trace (bb, trace);
|
|
|
315
|
|
|
316 bb = trace[0];
|
|
|
317 traced_insns += bb->frequency * counts [bb->index];
|
|
|
318 if (blocks[bb->index])
|
|
|
319 {
|
|
111
|
320 heap.delete_node (blocks[bb->index]);
|
|
0
|
321 blocks[bb->index] = NULL;
|
|
|
322 }
|
|
|
323
|
|
|
324 for (pos = 1; pos < n; pos++)
|
|
|
325 {
|
|
|
326 basic_block bb2 = trace[pos];
|
|
|
327
|
|
|
328 if (blocks[bb2->index])
|
|
|
329 {
|
|
111
|
330 heap.delete_node (blocks[bb2->index]);
|
|
0
|
331 blocks[bb2->index] = NULL;
|
|
|
332 }
|
|
|
333 traced_insns += bb2->frequency * counts [bb2->index];
|
|
|
334 if (EDGE_COUNT (bb2->preds) > 1
|
|
111
|
335 && can_duplicate_block_p (bb2)
|
|
|
336 /* We have the tendency to duplicate the loop header
|
|
|
337 of all do { } while loops. Do not do that - it is
|
|
|
338 not profitable and it might create a loop with multiple
|
|
|
339 entries or at least rotate the loop. */
|
|
|
340 && bb2->loop_father->header != bb2)
|
|
0
|
341 {
|
|
|
342 nduplicated += counts [bb2->index];
|
|
111
|
343 basic_block copy = transform_duplicate (bb, bb2);
|
|
0
|
344
|
|
|
345 /* Reconsider the original copy of block we've duplicated.
|
|
|
346 Removing the most common predecessor may make it to be
|
|
|
347 head. */
|
|
111
|
348 blocks[bb2->index] = heap.insert (-bb2->frequency, bb2);
|
|
0
|
349
|
|
|
350 if (dump_file)
|
|
|
351 fprintf (dump_file, "Duplicated %i as %i [%i]\n",
|
|
|
352 bb2->index, copy->index, copy->frequency);
|
|
|
353
|
|
|
354 bb2 = copy;
|
|
111
|
355 changed = true;
|
|
0
|
356 }
|
|
|
357 mark_bb_seen (bb2);
|
|
|
358 bb = bb2;
|
|
|
359 /* In case the trace became infrequent, stop duplicating. */
|
|
|
360 if (ignore_bb_p (bb))
|
|
|
361 break;
|
|
|
362 }
|
|
|
363 if (dump_file)
|
|
|
364 fprintf (dump_file, " covered now %.1f\n\n",
|
|
|
365 traced_insns * 100.0 / weighted_insns);
|
|
|
366 }
|
|
|
367 if (dump_file)
|
|
|
368 fprintf (dump_file, "Duplicated %i insns (%i%%)\n", nduplicated,
|
|
|
369 nduplicated * 100 / ninsns);
|
|
|
370
|
|
|
371 free_original_copy_tables ();
|
|
|
372 sbitmap_free (bb_seen);
|
|
|
373 free (trace);
|
|
|
374 free (counts);
|
|
111
|
375
|
|
|
376 return changed;
|
|
0
|
377 }
|
|
111
|
378
|
|
|
379 namespace {
|
|
0
|
380
|
|
111
|
381 const pass_data pass_data_tracer =
|
|
|
382 {
|
|
|
383 GIMPLE_PASS, /* type */
|
|
|
384 "tracer", /* name */
|
|
|
385 OPTGROUP_NONE, /* optinfo_flags */
|
|
|
386 TV_TRACER, /* tv_id */
|
|
|
387 0, /* properties_required */
|
|
|
388 0, /* properties_provided */
|
|
|
389 0, /* properties_destroyed */
|
|
|
390 0, /* todo_flags_start */
|
|
|
391 TODO_update_ssa, /* todo_flags_finish */
|
|
|
392 };
|
|
0
|
393
|
|
111
|
394 class pass_tracer : public gimple_opt_pass
|
|
0
|
395 {
|
|
111
|
396 public:
|
|
|
397 pass_tracer (gcc::context *ctxt)
|
|
|
398 : gimple_opt_pass (pass_data_tracer, ctxt)
|
|
|
399 {}
|
|
|
400
|
|
|
401 /* opt_pass methods: */
|
|
|
402 virtual bool gate (function *)
|
|
|
403 {
|
|
|
404 return (optimize > 0 && flag_tracer && flag_reorder_blocks);
|
|
|
405 }
|
|
0
|
406
|
|
111
|
407 virtual unsigned int execute (function *);
|
|
|
408
|
|
|
409 }; // class pass_tracer
|
|
|
410
|
|
|
411 unsigned int
|
|
|
412 pass_tracer::execute (function *fun)
|
|
|
413 {
|
|
|
414 bool changed;
|
|
|
415
|
|
|
416 if (n_basic_blocks_for_fn (fun) <= NUM_FIXED_BLOCKS + 1)
|
|
0
|
417 return 0;
|
|
|
418
|
|
|
419 mark_dfs_back_edges ();
|
|
|
420 if (dump_file)
|
|
111
|
421 brief_dump_cfg (dump_file, dump_flags);
|
|
0
|
422
|
|
|
423 /* Trace formation is done on the fly inside tail_duplicate */
|
|
111
|
424 changed = tail_duplicate ();
|
|
|
425 if (changed)
|
|
|
426 {
|
|
|
427 free_dominance_info (CDI_DOMINATORS);
|
|
|
428 /* If we changed the CFG schedule loops for fixup by cleanup_cfg. */
|
|
|
429 loops_state_set (LOOPS_NEED_FIXUP);
|
|
|
430 }
|
|
0
|
431
|
|
111
|
432 if (dump_file)
|
|
|
433 brief_dump_cfg (dump_file, dump_flags);
|
|
|
434
|
|
|
435 return changed ? TODO_cleanup_cfg : 0;
|
|
0
|
436 }
|
|
111
|
437 } // anon namespace
|
|
0
|
438
|
|
111
|
439 gimple_opt_pass *
|
|
|
440 make_pass_tracer (gcc::context *ctxt)
|
|
0
|
441 {
|
|
111
|
442 return new pass_tracer (ctxt);
|
|
|
443 }
|
