|
111
|
1 /* Read and annotate call graph profile from the auto profile data file.
|
|
131
|
2 Copyright (C) 2014-2018 Free Software Foundation, Inc.
|
|
111
|
3 Contributed by Dehao Chen (dehao@google.com)
|
|
|
4
|
|
|
5 This file is part of GCC.
|
|
|
6
|
|
|
7 GCC is free software; you can redistribute it and/or modify it under
|
|
|
8 the terms of the GNU General Public License as published by the Free
|
|
|
9 Software Foundation; either version 3, or (at your option) any later
|
|
|
10 version.
|
|
|
11
|
|
|
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with GCC; see the file COPYING3. If not see
|
|
|
19 <http://www.gnu.org/licenses/>. */
|
|
|
20
|
|
|
21 #include "config.h"
|
|
|
22 #define INCLUDE_MAP
|
|
|
23 #define INCLUDE_SET
|
|
|
24 #include "system.h"
|
|
|
25 #include "coretypes.h"
|
|
|
26 #include "backend.h"
|
|
|
27 #include "tree.h"
|
|
|
28 #include "gimple.h"
|
|
|
29 #include "predict.h"
|
|
|
30 #include "alloc-pool.h"
|
|
|
31 #include "tree-pass.h"
|
|
|
32 #include "ssa.h"
|
|
|
33 #include "cgraph.h"
|
|
|
34 #include "gcov-io.h"
|
|
|
35 #include "diagnostic-core.h"
|
|
|
36 #include "profile.h"
|
|
|
37 #include "langhooks.h"
|
|
|
38 #include "cfgloop.h"
|
|
|
39 #include "tree-cfg.h"
|
|
|
40 #include "tree-cfgcleanup.h"
|
|
|
41 #include "tree-into-ssa.h"
|
|
|
42 #include "gimple-iterator.h"
|
|
|
43 #include "value-prof.h"
|
|
|
44 #include "params.h"
|
|
|
45 #include "symbol-summary.h"
|
|
|
46 #include "ipa-prop.h"
|
|
|
47 #include "ipa-fnsummary.h"
|
|
|
48 #include "ipa-inline.h"
|
|
|
49 #include "tree-inline.h"
|
|
|
50 #include "auto-profile.h"
|
|
|
51 #include "tree-pretty-print.h"
|
|
|
52 #include "gimple-pretty-print.h"
|
|
|
53
|
|
|
54 /* The following routines implements AutoFDO optimization.
|
|
|
55
|
|
|
56 This optimization uses sampling profiles to annotate basic block counts
|
|
|
57 and uses heuristics to estimate branch probabilities.
|
|
|
58
|
|
|
59 There are three phases in AutoFDO:
|
|
|
60
|
|
|
61 Phase 1: Read profile from the profile data file.
|
|
|
62 The following info is read from the profile datafile:
|
|
|
63 * string_table: a map between function name and its index.
|
|
|
64 * autofdo_source_profile: a map from function_instance name to
|
|
|
65 function_instance. This is represented as a forest of
|
|
|
66 function_instances.
|
|
|
67 * WorkingSet: a histogram of how many instructions are covered for a
|
|
|
68 given percentage of total cycles. This is describing the binary
|
|
|
69 level information (not source level). This info is used to help
|
|
|
70 decide if we want aggressive optimizations that could increase
|
|
|
71 code footprint (e.g. loop unroll etc.)
|
|
|
72 A function instance is an instance of function that could either be a
|
|
|
73 standalone symbol, or a clone of a function that is inlined into another
|
|
|
74 function.
|
|
|
75
|
|
|
76 Phase 2: Early inline + value profile transformation.
|
|
|
77 Early inline uses autofdo_source_profile to find if a callsite is:
|
|
|
78 * inlined in the profiled binary.
|
|
|
79 * callee body is hot in the profiling run.
|
|
|
80 If both condition satisfies, early inline will inline the callsite
|
|
|
81 regardless of the code growth.
|
|
|
82 Phase 2 is an iterative process. During each iteration, we also check
|
|
|
83 if an indirect callsite is promoted and inlined in the profiling run.
|
|
|
84 If yes, vpt will happen to force promote it and in the next iteration,
|
|
|
85 einline will inline the promoted callsite in the next iteration.
|
|
|
86
|
|
|
87 Phase 3: Annotate control flow graph.
|
|
|
88 AutoFDO uses a separate pass to:
|
|
|
89 * Annotate basic block count
|
|
|
90 * Estimate branch probability
|
|
|
91
|
|
|
92 After the above 3 phases, all profile is readily annotated on the GCC IR.
|
|
|
93 AutoFDO tries to reuse all FDO infrastructure as much as possible to make
|
|
|
94 use of the profile. E.g. it uses existing mechanism to calculate the basic
|
|
|
95 block/edge frequency, as well as the cgraph node/edge count.
|
|
|
96 */
|
|
|
97
|
|
|
98 #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
|
|
|
99 #define AUTO_PROFILE_VERSION 1
|
|
|
100
|
|
|
101 namespace autofdo
|
|
|
102 {
|
|
|
103
|
|
|
104 /* Represent a source location: (function_decl, lineno). */
|
|
|
105 typedef std::pair<tree, unsigned> decl_lineno;
|
|
|
106
|
|
|
107 /* Represent an inline stack. vector[0] is the leaf node. */
|
|
|
108 typedef auto_vec<decl_lineno> inline_stack;
|
|
|
109
|
|
|
110 /* String array that stores function names. */
|
|
|
111 typedef auto_vec<char *> string_vector;
|
|
|
112
|
|
|
113 /* Map from function name's index in string_table to target's
|
|
|
114 execution count. */
|
|
|
115 typedef std::map<unsigned, gcov_type> icall_target_map;
|
|
|
116
|
|
|
117 /* Set of gimple stmts. Used to track if the stmt has already been promoted
|
|
|
118 to direct call. */
|
|
|
119 typedef std::set<gimple *> stmt_set;
|
|
|
120
|
|
|
121 /* Represent count info of an inline stack. */
|
|
|
122 struct count_info
|
|
|
123 {
|
|
|
124 /* Sampled count of the inline stack. */
|
|
|
125 gcov_type count;
|
|
|
126
|
|
|
127 /* Map from indirect call target to its sample count. */
|
|
|
128 icall_target_map targets;
|
|
|
129
|
|
|
130 /* Whether this inline stack is already used in annotation.
|
|
|
131
|
|
|
132 Each inline stack should only be used to annotate IR once.
|
|
|
133 This will be enforced when instruction-level discriminator
|
|
|
134 is supported. */
|
|
|
135 bool annotated;
|
|
|
136 };
|
|
|
137
|
|
|
138 /* operator< for "const char *". */
|
|
|
139 struct string_compare
|
|
|
140 {
|
|
|
141 bool operator()(const char *a, const char *b) const
|
|
|
142 {
|
|
|
143 return strcmp (a, b) < 0;
|
|
|
144 }
|
|
|
145 };
|
|
|
146
|
|
|
147 /* Store a string array, indexed by string position in the array. */
|
|
|
148 class string_table
|
|
|
149 {
|
|
|
150 public:
|
|
|
151 string_table ()
|
|
|
152 {}
|
|
|
153
|
|
|
154 ~string_table ();
|
|
|
155
|
|
|
156 /* For a given string, returns its index. */
|
|
|
157 int get_index (const char *name) const;
|
|
|
158
|
|
|
159 /* For a given decl, returns the index of the decl name. */
|
|
|
160 int get_index_by_decl (tree decl) const;
|
|
|
161
|
|
|
162 /* For a given index, returns the string. */
|
|
|
163 const char *get_name (int index) const;
|
|
|
164
|
|
|
165 /* Read profile, return TRUE on success. */
|
|
|
166 bool read ();
|
|
|
167
|
|
|
168 private:
|
|
|
169 typedef std::map<const char *, unsigned, string_compare> string_index_map;
|
|
|
170 string_vector vector_;
|
|
|
171 string_index_map map_;
|
|
|
172 };
|
|
|
173
|
|
|
174 /* Profile of a function instance:
|
|
|
175 1. total_count of the function.
|
|
|
176 2. head_count (entry basic block count) of the function (only valid when
|
|
|
177 function is a top-level function_instance, i.e. it is the original copy
|
|
|
178 instead of the inlined copy).
|
|
|
179 3. map from source location (decl_lineno) to profile (count_info).
|
|
|
180 4. map from callsite to callee function_instance. */
|
|
|
181 class function_instance
|
|
|
182 {
|
|
|
183 public:
|
|
|
184 typedef auto_vec<function_instance *> function_instance_stack;
|
|
|
185
|
|
|
186 /* Read the profile and return a function_instance with head count as
|
|
|
187 HEAD_COUNT. Recursively read callsites to create nested function_instances
|
|
|
188 too. STACK is used to track the recursive creation process. */
|
|
|
189 static function_instance *
|
|
|
190 read_function_instance (function_instance_stack *stack,
|
|
|
191 gcov_type head_count);
|
|
|
192
|
|
|
193 /* Recursively deallocate all callsites (nested function_instances). */
|
|
|
194 ~function_instance ();
|
|
|
195
|
|
|
196 /* Accessors. */
|
|
|
197 int
|
|
|
198 name () const
|
|
|
199 {
|
|
|
200 return name_;
|
|
|
201 }
|
|
|
202 gcov_type
|
|
|
203 total_count () const
|
|
|
204 {
|
|
|
205 return total_count_;
|
|
|
206 }
|
|
|
207 gcov_type
|
|
|
208 head_count () const
|
|
|
209 {
|
|
|
210 return head_count_;
|
|
|
211 }
|
|
|
212
|
|
|
213 /* Traverse callsites of the current function_instance to find one at the
|
|
|
214 location of LINENO and callee name represented in DECL. */
|
|
|
215 function_instance *get_function_instance_by_decl (unsigned lineno,
|
|
|
216 tree decl) const;
|
|
|
217
|
|
|
218 /* Store the profile info for LOC in INFO. Return TRUE if profile info
|
|
|
219 is found. */
|
|
|
220 bool get_count_info (location_t loc, count_info *info) const;
|
|
|
221
|
|
|
222 /* Read the inlined indirect call target profile for STMT and store it in
|
|
|
223 MAP, return the total count for all inlined indirect calls. */
|
|
|
224 gcov_type find_icall_target_map (gcall *stmt, icall_target_map *map) const;
|
|
|
225
|
|
|
226 /* Sum of counts that is used during annotation. */
|
|
|
227 gcov_type total_annotated_count () const;
|
|
|
228
|
|
|
229 /* Mark LOC as annotated. */
|
|
|
230 void mark_annotated (location_t loc);
|
|
|
231
|
|
|
232 private:
|
|
|
233 /* Callsite, represented as (decl_lineno, callee_function_name_index). */
|
|
|
234 typedef std::pair<unsigned, unsigned> callsite;
|
|
|
235
|
|
|
236 /* Map from callsite to callee function_instance. */
|
|
|
237 typedef std::map<callsite, function_instance *> callsite_map;
|
|
|
238
|
|
|
239 function_instance (unsigned name, gcov_type head_count)
|
|
|
240 : name_ (name), total_count_ (0), head_count_ (head_count)
|
|
|
241 {
|
|
|
242 }
|
|
|
243
|
|
|
244 /* Map from source location (decl_lineno) to profile (count_info). */
|
|
|
245 typedef std::map<unsigned, count_info> position_count_map;
|
|
|
246
|
|
|
247 /* function_instance name index in the string_table. */
|
|
|
248 unsigned name_;
|
|
|
249
|
|
|
250 /* Total sample count. */
|
|
|
251 gcov_type total_count_;
|
|
|
252
|
|
|
253 /* Entry BB's sample count. */
|
|
|
254 gcov_type head_count_;
|
|
|
255
|
|
|
256 /* Map from callsite location to callee function_instance. */
|
|
|
257 callsite_map callsites;
|
|
|
258
|
|
|
259 /* Map from source location to count_info. */
|
|
|
260 position_count_map pos_counts;
|
|
|
261 };
|
|
|
262
|
|
|
263 /* Profile for all functions. */
|
|
|
264 class autofdo_source_profile
|
|
|
265 {
|
|
|
266 public:
|
|
|
267 static autofdo_source_profile *
|
|
|
268 create ()
|
|
|
269 {
|
|
|
270 autofdo_source_profile *map = new autofdo_source_profile ();
|
|
|
271
|
|
|
272 if (map->read ())
|
|
|
273 return map;
|
|
|
274 delete map;
|
|
|
275 return NULL;
|
|
|
276 }
|
|
|
277
|
|
|
278 ~autofdo_source_profile ();
|
|
|
279
|
|
|
280 /* For a given DECL, returns the top-level function_instance. */
|
|
|
281 function_instance *get_function_instance_by_decl (tree decl) const;
|
|
|
282
|
|
|
283 /* Find count_info for a given gimple STMT. If found, store the count_info
|
|
|
284 in INFO and return true; otherwise return false. */
|
|
|
285 bool get_count_info (gimple *stmt, count_info *info) const;
|
|
|
286
|
|
|
287 /* Find total count of the callee of EDGE. */
|
|
|
288 gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;
|
|
|
289
|
|
|
290 /* Update value profile INFO for STMT from the inlined indirect callsite.
|
|
|
291 Return true if INFO is updated. */
|
|
|
292 bool update_inlined_ind_target (gcall *stmt, count_info *info);
|
|
|
293
|
|
|
294 /* Mark LOC as annotated. */
|
|
|
295 void mark_annotated (location_t loc);
|
|
|
296
|
|
|
297 private:
|
|
|
298 /* Map from function_instance name index (in string_table) to
|
|
|
299 function_instance. */
|
|
|
300 typedef std::map<unsigned, function_instance *> name_function_instance_map;
|
|
|
301
|
|
|
302 autofdo_source_profile () {}
|
|
|
303
|
|
|
304 /* Read AutoFDO profile and returns TRUE on success. */
|
|
|
305 bool read ();
|
|
|
306
|
|
|
307 /* Return the function_instance in the profile that correspond to the
|
|
|
308 inline STACK. */
|
|
|
309 function_instance *
|
|
|
310 get_function_instance_by_inline_stack (const inline_stack &stack) const;
|
|
|
311
|
|
|
312 name_function_instance_map map_;
|
|
|
313 };
|
|
|
314
|
|
|
315 /* Store the strings read from the profile data file. */
|
|
|
316 static string_table *afdo_string_table;
|
|
|
317
|
|
|
318 /* Store the AutoFDO source profile. */
|
|
|
319 static autofdo_source_profile *afdo_source_profile;
|
|
|
320
|
|
131
|
321 /* gcov_summary structure to store the profile_info. */
|
|
|
322 static gcov_summary *afdo_profile_info;
|
|
111
|
323
|
|
|
324 /* Helper functions. */
|
|
|
325
|
|
|
326 /* Return the original name of NAME: strip the suffix that starts
|
|
|
327 with '.' Caller is responsible for freeing RET. */
|
|
|
328
|
|
|
329 static char *
|
|
|
330 get_original_name (const char *name)
|
|
|
331 {
|
|
|
332 char *ret = xstrdup (name);
|
|
|
333 char *find = strchr (ret, '.');
|
|
|
334 if (find != NULL)
|
|
|
335 *find = 0;
|
|
|
336 return ret;
|
|
|
337 }
|
|
|
338
|
|
|
339 /* Return the combined location, which is a 32bit integer in which
|
|
|
340 higher 16 bits stores the line offset of LOC to the start lineno
|
|
|
341 of DECL, The lower 16 bits stores the discriminator. */
|
|
|
342
|
|
|
343 static unsigned
|
|
|
344 get_combined_location (location_t loc, tree decl)
|
|
|
345 {
|
|
|
346 /* TODO: allow more bits for line and less bits for discriminator. */
|
|
|
347 if (LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl) >= (1<<16))
|
|
|
348 warning_at (loc, OPT_Woverflow, "offset exceeds 16 bytes");
|
|
|
349 return ((LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16);
|
|
|
350 }
|
|
|
351
|
|
|
352 /* Return the function decl of a given lexical BLOCK. */
|
|
|
353
|
|
|
354 static tree
|
|
|
355 get_function_decl_from_block (tree block)
|
|
|
356 {
|
|
131
|
357 if (!inlined_function_outer_scope_p (block))
|
|
111
|
358 return NULL_TREE;
|
|
|
359
|
|
131
|
360 return BLOCK_ABSTRACT_ORIGIN (block);
|
|
111
|
361 }
|
|
|
362
|
|
|
363 /* Store inline stack for STMT in STACK. */
|
|
|
364
|
|
|
365 static void
|
|
|
366 get_inline_stack (location_t locus, inline_stack *stack)
|
|
|
367 {
|
|
|
368 if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
|
|
|
369 return;
|
|
|
370
|
|
|
371 tree block = LOCATION_BLOCK (locus);
|
|
|
372 if (block && TREE_CODE (block) == BLOCK)
|
|
|
373 {
|
|
|
374 int level = 0;
|
|
|
375 for (block = BLOCK_SUPERCONTEXT (block);
|
|
|
376 block && (TREE_CODE (block) == BLOCK);
|
|
|
377 block = BLOCK_SUPERCONTEXT (block))
|
|
|
378 {
|
|
|
379 location_t tmp_locus = BLOCK_SOURCE_LOCATION (block);
|
|
|
380 if (LOCATION_LOCUS (tmp_locus) == UNKNOWN_LOCATION)
|
|
|
381 continue;
|
|
|
382
|
|
|
383 tree decl = get_function_decl_from_block (block);
|
|
|
384 stack->safe_push (
|
|
|
385 std::make_pair (decl, get_combined_location (locus, decl)));
|
|
|
386 locus = tmp_locus;
|
|
|
387 level++;
|
|
|
388 }
|
|
|
389 }
|
|
|
390 stack->safe_push (
|
|
|
391 std::make_pair (current_function_decl,
|
|
|
392 get_combined_location (locus, current_function_decl)));
|
|
|
393 }
|
|
|
394
|
|
|
395 /* Return STMT's combined location, which is a 32bit integer in which
|
|
|
396 higher 16 bits stores the line offset of LOC to the start lineno
|
|
|
397 of DECL, The lower 16 bits stores the discriminator. */
|
|
|
398
|
|
|
399 static unsigned
|
|
|
400 get_relative_location_for_stmt (gimple *stmt)
|
|
|
401 {
|
|
|
402 location_t locus = gimple_location (stmt);
|
|
|
403 if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
|
|
|
404 return UNKNOWN_LOCATION;
|
|
|
405
|
|
|
406 for (tree block = gimple_block (stmt); block && (TREE_CODE (block) == BLOCK);
|
|
|
407 block = BLOCK_SUPERCONTEXT (block))
|
|
|
408 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) != UNKNOWN_LOCATION)
|
|
|
409 return get_combined_location (locus,
|
|
|
410 get_function_decl_from_block (block));
|
|
|
411 return get_combined_location (locus, current_function_decl);
|
|
|
412 }
|
|
|
413
|
|
|
414 /* Return true if BB contains indirect call. */
|
|
|
415
|
|
|
416 static bool
|
|
|
417 has_indirect_call (basic_block bb)
|
|
|
418 {
|
|
|
419 gimple_stmt_iterator gsi;
|
|
|
420
|
|
|
421 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
422 {
|
|
|
423 gimple *stmt = gsi_stmt (gsi);
|
|
|
424 if (gimple_code (stmt) == GIMPLE_CALL && !gimple_call_internal_p (stmt)
|
|
|
425 && (gimple_call_fn (stmt) == NULL
|
|
|
426 || TREE_CODE (gimple_call_fn (stmt)) != FUNCTION_DECL))
|
|
|
427 return true;
|
|
|
428 }
|
|
|
429 return false;
|
|
|
430 }
|
|
|
431
|
|
|
432 /* Member functions for string_table. */
|
|
|
433
|
|
|
434 /* Deconstructor. */
|
|
|
435
|
|
|
436 string_table::~string_table ()
|
|
|
437 {
|
|
|
438 for (unsigned i = 0; i < vector_.length (); i++)
|
|
|
439 free (vector_[i]);
|
|
|
440 }
|
|
|
441
|
|
|
442
|
|
|
443 /* Return the index of a given function NAME. Return -1 if NAME is not
|
|
|
444 found in string table. */
|
|
|
445
|
|
|
446 int
|
|
|
447 string_table::get_index (const char *name) const
|
|
|
448 {
|
|
|
449 if (name == NULL)
|
|
|
450 return -1;
|
|
|
451 string_index_map::const_iterator iter = map_.find (name);
|
|
|
452 if (iter == map_.end ())
|
|
|
453 return -1;
|
|
|
454
|
|
|
455 return iter->second;
|
|
|
456 }
|
|
|
457
|
|
|
458 /* Return the index of a given function DECL. Return -1 if DECL is not
|
|
|
459 found in string table. */
|
|
|
460
|
|
|
461 int
|
|
|
462 string_table::get_index_by_decl (tree decl) const
|
|
|
463 {
|
|
|
464 char *name
|
|
|
465 = get_original_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
|
|
|
466 int ret = get_index (name);
|
|
|
467 free (name);
|
|
|
468 if (ret != -1)
|
|
|
469 return ret;
|
|
|
470 ret = get_index (lang_hooks.dwarf_name (decl, 0));
|
|
|
471 if (ret != -1)
|
|
|
472 return ret;
|
|
131
|
473 if (DECL_ABSTRACT_ORIGIN (decl) && DECL_ABSTRACT_ORIGIN (decl) != decl)
|
|
111
|
474 return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl));
|
|
|
475
|
|
|
476 return -1;
|
|
|
477 }
|
|
|
478
|
|
|
479 /* Return the function name of a given INDEX. */
|
|
|
480
|
|
|
481 const char *
|
|
|
482 string_table::get_name (int index) const
|
|
|
483 {
|
|
|
484 gcc_assert (index > 0 && index < (int)vector_.length ());
|
|
|
485 return vector_[index];
|
|
|
486 }
|
|
|
487
|
|
|
488 /* Read the string table. Return TRUE if reading is successful. */
|
|
|
489
|
|
|
490 bool
|
|
|
491 string_table::read ()
|
|
|
492 {
|
|
|
493 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES)
|
|
|
494 return false;
|
|
|
495 /* Skip the length of the section. */
|
|
|
496 gcov_read_unsigned ();
|
|
|
497 /* Read in the file name table. */
|
|
|
498 unsigned string_num = gcov_read_unsigned ();
|
|
|
499 for (unsigned i = 0; i < string_num; i++)
|
|
|
500 {
|
|
|
501 vector_.safe_push (get_original_name (gcov_read_string ()));
|
|
|
502 map_[vector_.last ()] = i;
|
|
|
503 }
|
|
|
504 return true;
|
|
|
505 }
|
|
|
506
|
|
|
507 /* Member functions for function_instance. */
|
|
|
508
|
|
|
509 function_instance::~function_instance ()
|
|
|
510 {
|
|
|
511 for (callsite_map::iterator iter = callsites.begin ();
|
|
|
512 iter != callsites.end (); ++iter)
|
|
|
513 delete iter->second;
|
|
|
514 }
|
|
|
515
|
|
|
516 /* Traverse callsites of the current function_instance to find one at the
|
|
|
517 location of LINENO and callee name represented in DECL. */
|
|
|
518
|
|
|
519 function_instance *
|
|
|
520 function_instance::get_function_instance_by_decl (unsigned lineno,
|
|
|
521 tree decl) const
|
|
|
522 {
|
|
|
523 int func_name_idx = afdo_string_table->get_index_by_decl (decl);
|
|
|
524 if (func_name_idx != -1)
|
|
|
525 {
|
|
|
526 callsite_map::const_iterator ret
|
|
|
527 = callsites.find (std::make_pair (lineno, func_name_idx));
|
|
|
528 if (ret != callsites.end ())
|
|
|
529 return ret->second;
|
|
|
530 }
|
|
|
531 func_name_idx
|
|
|
532 = afdo_string_table->get_index (lang_hooks.dwarf_name (decl, 0));
|
|
|
533 if (func_name_idx != -1)
|
|
|
534 {
|
|
|
535 callsite_map::const_iterator ret
|
|
|
536 = callsites.find (std::make_pair (lineno, func_name_idx));
|
|
|
537 if (ret != callsites.end ())
|
|
|
538 return ret->second;
|
|
|
539 }
|
|
|
540 if (DECL_ABSTRACT_ORIGIN (decl))
|
|
|
541 return get_function_instance_by_decl (lineno, DECL_ABSTRACT_ORIGIN (decl));
|
|
|
542
|
|
|
543 return NULL;
|
|
|
544 }
|
|
|
545
|
|
|
546 /* Store the profile info for LOC in INFO. Return TRUE if profile info
|
|
|
547 is found. */
|
|
|
548
|
|
|
549 bool
|
|
|
550 function_instance::get_count_info (location_t loc, count_info *info) const
|
|
|
551 {
|
|
|
552 position_count_map::const_iterator iter = pos_counts.find (loc);
|
|
|
553 if (iter == pos_counts.end ())
|
|
|
554 return false;
|
|
|
555 *info = iter->second;
|
|
|
556 return true;
|
|
|
557 }
|
|
|
558
|
|
|
559 /* Mark LOC as annotated. */
|
|
|
560
|
|
|
561 void
|
|
|
562 function_instance::mark_annotated (location_t loc)
|
|
|
563 {
|
|
|
564 position_count_map::iterator iter = pos_counts.find (loc);
|
|
|
565 if (iter == pos_counts.end ())
|
|
|
566 return;
|
|
|
567 iter->second.annotated = true;
|
|
|
568 }
|
|
|
569
|
|
|
570 /* Read the inlined indirect call target profile for STMT and store it in
|
|
|
571 MAP, return the total count for all inlined indirect calls. */
|
|
|
572
|
|
|
573 gcov_type
|
|
|
574 function_instance::find_icall_target_map (gcall *stmt,
|
|
|
575 icall_target_map *map) const
|
|
|
576 {
|
|
|
577 gcov_type ret = 0;
|
|
|
578 unsigned stmt_offset = get_relative_location_for_stmt (stmt);
|
|
|
579
|
|
|
580 for (callsite_map::const_iterator iter = callsites.begin ();
|
|
|
581 iter != callsites.end (); ++iter)
|
|
|
582 {
|
|
|
583 unsigned callee = iter->second->name ();
|
|
|
584 /* Check if callsite location match the stmt. */
|
|
|
585 if (iter->first.first != stmt_offset)
|
|
|
586 continue;
|
|
|
587 struct cgraph_node *node = cgraph_node::get_for_asmname (
|
|
|
588 get_identifier (afdo_string_table->get_name (callee)));
|
|
|
589 if (node == NULL)
|
|
|
590 continue;
|
|
|
591 if (!check_ic_target (stmt, node))
|
|
|
592 continue;
|
|
|
593 (*map)[callee] = iter->second->total_count ();
|
|
|
594 ret += iter->second->total_count ();
|
|
|
595 }
|
|
|
596 return ret;
|
|
|
597 }
|
|
|
598
|
|
|
599 /* Read the profile and create a function_instance with head count as
|
|
|
600 HEAD_COUNT. Recursively read callsites to create nested function_instances
|
|
|
601 too. STACK is used to track the recursive creation process. */
|
|
|
602
|
|
|
603 /* function instance profile format:
|
|
|
604
|
|
|
605 ENTRY_COUNT: 8 bytes
|
|
|
606 NAME_INDEX: 4 bytes
|
|
|
607 NUM_POS_COUNTS: 4 bytes
|
|
|
608 NUM_CALLSITES: 4 byte
|
|
|
609 POS_COUNT_1:
|
|
|
610 POS_1_OFFSET: 4 bytes
|
|
|
611 NUM_TARGETS: 4 bytes
|
|
|
612 COUNT: 8 bytes
|
|
|
613 TARGET_1:
|
|
|
614 VALUE_PROFILE_TYPE: 4 bytes
|
|
|
615 TARGET_IDX: 8 bytes
|
|
|
616 COUNT: 8 bytes
|
|
|
617 TARGET_2
|
|
|
618 ...
|
|
|
619 TARGET_n
|
|
|
620 POS_COUNT_2
|
|
|
621 ...
|
|
|
622 POS_COUNT_N
|
|
|
623 CALLSITE_1:
|
|
|
624 CALLSITE_1_OFFSET: 4 bytes
|
|
|
625 FUNCTION_INSTANCE_PROFILE (nested)
|
|
|
626 CALLSITE_2
|
|
|
627 ...
|
|
|
628 CALLSITE_n. */
|
|
|
629
|
|
|
630 function_instance *
|
|
|
631 function_instance::read_function_instance (function_instance_stack *stack,
|
|
|
632 gcov_type head_count)
|
|
|
633 {
|
|
|
634 unsigned name = gcov_read_unsigned ();
|
|
|
635 unsigned num_pos_counts = gcov_read_unsigned ();
|
|
|
636 unsigned num_callsites = gcov_read_unsigned ();
|
|
|
637 function_instance *s = new function_instance (name, head_count);
|
|
|
638 stack->safe_push (s);
|
|
|
639
|
|
|
640 for (unsigned i = 0; i < num_pos_counts; i++)
|
|
|
641 {
|
|
|
642 unsigned offset = gcov_read_unsigned () & 0xffff0000;
|
|
|
643 unsigned num_targets = gcov_read_unsigned ();
|
|
|
644 gcov_type count = gcov_read_counter ();
|
|
|
645 s->pos_counts[offset].count = count;
|
|
|
646 for (unsigned j = 0; j < stack->length (); j++)
|
|
|
647 (*stack)[j]->total_count_ += count;
|
|
|
648 for (unsigned j = 0; j < num_targets; j++)
|
|
|
649 {
|
|
|
650 /* Only indirect call target histogram is supported now. */
|
|
|
651 gcov_read_unsigned ();
|
|
|
652 gcov_type target_idx = gcov_read_counter ();
|
|
|
653 s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
|
|
|
654 }
|
|
|
655 }
|
|
|
656 for (unsigned i = 0; i < num_callsites; i++)
|
|
|
657 {
|
|
|
658 unsigned offset = gcov_read_unsigned ();
|
|
|
659 function_instance *callee_function_instance
|
|
|
660 = read_function_instance (stack, 0);
|
|
|
661 s->callsites[std::make_pair (offset, callee_function_instance->name ())]
|
|
|
662 = callee_function_instance;
|
|
|
663 }
|
|
|
664 stack->pop ();
|
|
|
665 return s;
|
|
|
666 }
|
|
|
667
|
|
|
668 /* Sum of counts that is used during annotation. */
|
|
|
669
|
|
|
670 gcov_type
|
|
|
671 function_instance::total_annotated_count () const
|
|
|
672 {
|
|
|
673 gcov_type ret = 0;
|
|
|
674 for (callsite_map::const_iterator iter = callsites.begin ();
|
|
|
675 iter != callsites.end (); ++iter)
|
|
|
676 ret += iter->second->total_annotated_count ();
|
|
|
677 for (position_count_map::const_iterator iter = pos_counts.begin ();
|
|
|
678 iter != pos_counts.end (); ++iter)
|
|
|
679 if (iter->second.annotated)
|
|
|
680 ret += iter->second.count;
|
|
|
681 return ret;
|
|
|
682 }
|
|
|
683
|
|
|
684 /* Member functions for autofdo_source_profile. */
|
|
|
685
|
|
|
686 autofdo_source_profile::~autofdo_source_profile ()
|
|
|
687 {
|
|
|
688 for (name_function_instance_map::const_iterator iter = map_.begin ();
|
|
|
689 iter != map_.end (); ++iter)
|
|
|
690 delete iter->second;
|
|
|
691 }
|
|
|
692
|
|
|
693 /* For a given DECL, returns the top-level function_instance. */
|
|
|
694
|
|
|
695 function_instance *
|
|
|
696 autofdo_source_profile::get_function_instance_by_decl (tree decl) const
|
|
|
697 {
|
|
|
698 int index = afdo_string_table->get_index_by_decl (decl);
|
|
|
699 if (index == -1)
|
|
|
700 return NULL;
|
|
|
701 name_function_instance_map::const_iterator ret = map_.find (index);
|
|
|
702 return ret == map_.end () ? NULL : ret->second;
|
|
|
703 }
|
|
|
704
|
|
|
705 /* Find count_info for a given gimple STMT. If found, store the count_info
|
|
|
706 in INFO and return true; otherwise return false. */
|
|
|
707
|
|
|
708 bool
|
|
|
709 autofdo_source_profile::get_count_info (gimple *stmt, count_info *info) const
|
|
|
710 {
|
|
|
711 if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
|
|
|
712 return false;
|
|
|
713
|
|
|
714 inline_stack stack;
|
|
|
715 get_inline_stack (gimple_location (stmt), &stack);
|
|
|
716 if (stack.length () == 0)
|
|
|
717 return false;
|
|
|
718 function_instance *s = get_function_instance_by_inline_stack (stack);
|
|
|
719 if (s == NULL)
|
|
|
720 return false;
|
|
|
721 return s->get_count_info (stack[0].second, info);
|
|
|
722 }
|
|
|
723
|
|
|
724 /* Mark LOC as annotated. */
|
|
|
725
|
|
|
726 void
|
|
|
727 autofdo_source_profile::mark_annotated (location_t loc)
|
|
|
728 {
|
|
|
729 inline_stack stack;
|
|
|
730 get_inline_stack (loc, &stack);
|
|
|
731 if (stack.length () == 0)
|
|
|
732 return;
|
|
|
733 function_instance *s = get_function_instance_by_inline_stack (stack);
|
|
|
734 if (s == NULL)
|
|
|
735 return;
|
|
|
736 s->mark_annotated (stack[0].second);
|
|
|
737 }
|
|
|
738
|
|
|
739 /* Update value profile INFO for STMT from the inlined indirect callsite.
|
|
|
740 Return true if INFO is updated. */
|
|
|
741
|
|
|
742 bool
|
|
|
743 autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
|
|
|
744 count_info *info)
|
|
|
745 {
|
|
|
746 if (dump_file)
|
|
|
747 {
|
|
|
748 fprintf (dump_file, "Checking indirect call -> direct call ");
|
|
|
749 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
|
|
|
750 }
|
|
|
751
|
|
|
752 if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
|
|
|
753 {
|
|
|
754 if (dump_file)
|
|
|
755 fprintf (dump_file, " good locus\n");
|
|
|
756 return false;
|
|
|
757 }
|
|
|
758
|
|
|
759 count_info old_info;
|
|
|
760 get_count_info (stmt, &old_info);
|
|
|
761 gcov_type total = 0;
|
|
|
762 for (icall_target_map::const_iterator iter = old_info.targets.begin ();
|
|
|
763 iter != old_info.targets.end (); ++iter)
|
|
|
764 total += iter->second;
|
|
|
765
|
|
|
766 /* Program behavior changed, original promoted (and inlined) target is not
|
|
|
767 hot any more. Will avoid promote the original target.
|
|
|
768
|
|
|
769 To check if original promoted target is still hot, we check the total
|
|
|
770 count of the unpromoted targets (stored in TOTAL). If a callsite count
|
|
|
771 (stored in INFO) is smaller than half of the total count, the original
|
|
|
772 promoted target is considered not hot any more. */
|
|
|
773 if (info->count < total / 2)
|
|
|
774 {
|
|
|
775 if (dump_file)
|
|
|
776 fprintf (dump_file, " not hot anymore %ld < %ld",
|
|
|
777 (long)info->count,
|
|
|
778 (long)total /2);
|
|
|
779 return false;
|
|
|
780 }
|
|
|
781
|
|
|
782 inline_stack stack;
|
|
|
783 get_inline_stack (gimple_location (stmt), &stack);
|
|
|
784 if (stack.length () == 0)
|
|
|
785 {
|
|
|
786 if (dump_file)
|
|
|
787 fprintf (dump_file, " no inline stack\n");
|
|
|
788 return false;
|
|
|
789 }
|
|
|
790 function_instance *s = get_function_instance_by_inline_stack (stack);
|
|
|
791 if (s == NULL)
|
|
|
792 {
|
|
|
793 if (dump_file)
|
|
|
794 fprintf (dump_file, " function not found in inline stack\n");
|
|
|
795 return false;
|
|
|
796 }
|
|
|
797 icall_target_map map;
|
|
|
798 if (s->find_icall_target_map (stmt, &map) == 0)
|
|
|
799 {
|
|
|
800 if (dump_file)
|
|
|
801 fprintf (dump_file, " no target map\n");
|
|
|
802 return false;
|
|
|
803 }
|
|
|
804 for (icall_target_map::const_iterator iter = map.begin ();
|
|
|
805 iter != map.end (); ++iter)
|
|
|
806 info->targets[iter->first] = iter->second;
|
|
|
807 if (dump_file)
|
|
|
808 fprintf (dump_file, " looks good\n");
|
|
|
809 return true;
|
|
|
810 }
|
|
|
811
|
|
|
812 /* Find total count of the callee of EDGE. */
|
|
|
813
|
|
|
814 gcov_type
|
|
|
815 autofdo_source_profile::get_callsite_total_count (
|
|
|
816 struct cgraph_edge *edge) const
|
|
|
817 {
|
|
|
818 inline_stack stack;
|
|
|
819 stack.safe_push (std::make_pair (edge->callee->decl, 0));
|
|
|
820 get_inline_stack (gimple_location (edge->call_stmt), &stack);
|
|
|
821
|
|
|
822 function_instance *s = get_function_instance_by_inline_stack (stack);
|
|
|
823 if (s == NULL
|
|
|
824 || afdo_string_table->get_index (IDENTIFIER_POINTER (
|
|
|
825 DECL_ASSEMBLER_NAME (edge->callee->decl))) != s->name ())
|
|
|
826 return 0;
|
|
|
827
|
|
|
828 return s->total_count ();
|
|
|
829 }
|
|
|
830
|
|
|
831 /* Read AutoFDO profile and returns TRUE on success. */
|
|
|
832
|
|
|
833 /* source profile format:
|
|
|
834
|
|
|
835 GCOV_TAG_AFDO_FUNCTION: 4 bytes
|
|
|
836 LENGTH: 4 bytes
|
|
|
837 NUM_FUNCTIONS: 4 bytes
|
|
|
838 FUNCTION_INSTANCE_1
|
|
|
839 FUNCTION_INSTANCE_2
|
|
|
840 ...
|
|
|
841 FUNCTION_INSTANCE_N. */
|
|
|
842
|
|
|
843 bool
|
|
|
844 autofdo_source_profile::read ()
|
|
|
845 {
|
|
|
846 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION)
|
|
|
847 {
|
|
131
|
848 inform (UNKNOWN_LOCATION, "Not expected TAG.");
|
|
111
|
849 return false;
|
|
|
850 }
|
|
|
851
|
|
|
852 /* Skip the length of the section. */
|
|
|
853 gcov_read_unsigned ();
|
|
|
854
|
|
|
855 /* Read in the function/callsite profile, and store it in local
|
|
|
856 data structure. */
|
|
|
857 unsigned function_num = gcov_read_unsigned ();
|
|
|
858 for (unsigned i = 0; i < function_num; i++)
|
|
|
859 {
|
|
|
860 function_instance::function_instance_stack stack;
|
|
|
861 function_instance *s = function_instance::read_function_instance (
|
|
|
862 &stack, gcov_read_counter ());
|
|
|
863 map_[s->name ()] = s;
|
|
|
864 }
|
|
|
865 return true;
|
|
|
866 }
|
|
|
867
|
|
|
868 /* Return the function_instance in the profile that correspond to the
|
|
|
869 inline STACK. */
|
|
|
870
|
|
|
871 function_instance *
|
|
|
872 autofdo_source_profile::get_function_instance_by_inline_stack (
|
|
|
873 const inline_stack &stack) const
|
|
|
874 {
|
|
|
875 name_function_instance_map::const_iterator iter = map_.find (
|
|
|
876 afdo_string_table->get_index_by_decl (stack[stack.length () - 1].first));
|
|
|
877 if (iter == map_.end())
|
|
|
878 return NULL;
|
|
|
879 function_instance *s = iter->second;
|
|
|
880 for (unsigned i = stack.length() - 1; i > 0; i--)
|
|
|
881 {
|
|
|
882 s = s->get_function_instance_by_decl (
|
|
|
883 stack[i].second, stack[i - 1].first);
|
|
|
884 if (s == NULL)
|
|
|
885 return NULL;
|
|
|
886 }
|
|
|
887 return s;
|
|
|
888 }
|
|
|
889
|
|
|
890 /* Module profile is only used by LIPO. Here we simply ignore it. */
|
|
|
891
|
|
|
892 static void
|
|
|
893 fake_read_autofdo_module_profile ()
|
|
|
894 {
|
|
|
895 /* Read in the module info. */
|
|
|
896 gcov_read_unsigned ();
|
|
|
897
|
|
|
898 /* Skip the length of the section. */
|
|
|
899 gcov_read_unsigned ();
|
|
|
900
|
|
|
901 /* Read in the file name table. */
|
|
|
902 unsigned total_module_num = gcov_read_unsigned ();
|
|
|
903 gcc_assert (total_module_num == 0);
|
|
|
904 }
|
|
|
905
|
|
|
906 /* Read data from profile data file. */
|
|
|
907
|
|
|
908 static void
|
|
|
909 read_profile (void)
|
|
|
910 {
|
|
|
911 if (gcov_open (auto_profile_file, 1) == 0)
|
|
|
912 {
|
|
|
913 error ("cannot open profile file %s", auto_profile_file);
|
|
|
914 return;
|
|
|
915 }
|
|
|
916
|
|
|
917 if (gcov_read_unsigned () != GCOV_DATA_MAGIC)
|
|
|
918 {
|
|
|
919 error ("AutoFDO profile magic number does not match");
|
|
|
920 return;
|
|
|
921 }
|
|
|
922
|
|
|
923 /* Skip the version number. */
|
|
|
924 unsigned version = gcov_read_unsigned ();
|
|
|
925 if (version != AUTO_PROFILE_VERSION)
|
|
|
926 {
|
|
|
927 error ("AutoFDO profile version %u does match %u",
|
|
|
928 version, AUTO_PROFILE_VERSION);
|
|
|
929 return;
|
|
|
930 }
|
|
|
931
|
|
|
932 /* Skip the empty integer. */
|
|
|
933 gcov_read_unsigned ();
|
|
|
934
|
|
|
935 /* string_table. */
|
|
|
936 afdo_string_table = new string_table ();
|
|
|
937 if (!afdo_string_table->read())
|
|
|
938 {
|
|
|
939 error ("cannot read string table from %s", auto_profile_file);
|
|
|
940 return;
|
|
|
941 }
|
|
|
942
|
|
|
943 /* autofdo_source_profile. */
|
|
|
944 afdo_source_profile = autofdo_source_profile::create ();
|
|
|
945 if (afdo_source_profile == NULL)
|
|
|
946 {
|
|
|
947 error ("cannot read function profile from %s", auto_profile_file);
|
|
|
948 return;
|
|
|
949 }
|
|
|
950
|
|
|
951 /* autofdo_module_profile. */
|
|
|
952 fake_read_autofdo_module_profile ();
|
|
|
953 }
|
|
|
954
|
|
|
955 /* From AutoFDO profiles, find values inside STMT for that we want to measure
|
|
|
956 histograms for indirect-call optimization.
|
|
|
957
|
|
|
958 This function is actually served for 2 purposes:
|
|
|
959 * before annotation, we need to mark histogram, promote and inline
|
|
|
960 * after annotation, we just need to mark, and let follow-up logic to
|
|
|
961 decide if it needs to promote and inline. */
|
|
|
962
|
|
|
963 static void
|
|
|
964 afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,
|
|
|
965 bool transform)
|
|
|
966 {
|
|
|
967 gimple *gs = gsi_stmt (*gsi);
|
|
|
968 tree callee;
|
|
|
969
|
|
|
970 if (map.size () == 0)
|
|
|
971 return;
|
|
|
972 gcall *stmt = dyn_cast <gcall *> (gs);
|
|
|
973 if ((!stmt) || gimple_call_fndecl (stmt) != NULL_TREE)
|
|
|
974 return;
|
|
|
975
|
|
|
976 callee = gimple_call_fn (stmt);
|
|
|
977
|
|
|
978 histogram_value hist = gimple_alloc_histogram_value (
|
|
|
979 cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
|
|
|
980 hist->n_counters = 3;
|
|
|
981 hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
|
|
|
982 gimple_add_histogram_value (cfun, stmt, hist);
|
|
|
983
|
|
|
984 gcov_type total = 0;
|
|
|
985 icall_target_map::const_iterator max_iter = map.end ();
|
|
|
986
|
|
|
987 for (icall_target_map::const_iterator iter = map.begin ();
|
|
|
988 iter != map.end (); ++iter)
|
|
|
989 {
|
|
|
990 total += iter->second;
|
|
|
991 if (max_iter == map.end () || max_iter->second < iter->second)
|
|
|
992 max_iter = iter;
|
|
|
993 }
|
|
|
994
|
|
|
995 hist->hvalue.counters[0]
|
|
|
996 = (unsigned long long)afdo_string_table->get_name (max_iter->first);
|
|
|
997 hist->hvalue.counters[1] = max_iter->second;
|
|
|
998 hist->hvalue.counters[2] = total;
|
|
|
999
|
|
|
1000 if (!transform)
|
|
|
1001 return;
|
|
|
1002
|
|
|
1003 struct cgraph_edge *indirect_edge
|
|
|
1004 = cgraph_node::get (current_function_decl)->get_edge (stmt);
|
|
|
1005 struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
|
|
|
1006 get_identifier ((const char *) hist->hvalue.counters[0]));
|
|
|
1007
|
|
|
1008 if (dump_file)
|
|
|
1009 {
|
|
|
1010 fprintf (dump_file, "Indirect call -> direct call ");
|
|
|
1011 print_generic_expr (dump_file, callee, TDF_SLIM);
|
|
|
1012 fprintf (dump_file, " => ");
|
|
|
1013 print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
|
|
|
1014 }
|
|
|
1015
|
|
|
1016 if (direct_call == NULL || !check_ic_target (stmt, direct_call))
|
|
|
1017 {
|
|
|
1018 if (dump_file)
|
|
|
1019 fprintf (dump_file, " not transforming\n");
|
|
|
1020 return;
|
|
|
1021 }
|
|
|
1022 if (DECL_STRUCT_FUNCTION (direct_call->decl) == NULL)
|
|
|
1023 {
|
|
|
1024 if (dump_file)
|
|
|
1025 fprintf (dump_file, " no declaration\n");
|
|
|
1026 return;
|
|
|
1027 }
|
|
|
1028
|
|
|
1029 if (dump_file)
|
|
|
1030 {
|
|
|
1031 fprintf (dump_file, " transformation on insn ");
|
|
|
1032 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
|
|
|
1033 fprintf (dump_file, "\n");
|
|
|
1034 }
|
|
|
1035
|
|
|
1036 /* FIXME: Count should be initialized. */
|
|
|
1037 struct cgraph_edge *new_edge
|
|
|
1038 = indirect_edge->make_speculative (direct_call,
|
|
131
|
1039 profile_count::uninitialized ());
|
|
111
|
1040 new_edge->redirect_call_stmt_to_callee ();
|
|
|
1041 gimple_remove_histogram_value (cfun, stmt, hist);
|
|
|
1042 inline_call (new_edge, true, NULL, NULL, false);
|
|
|
1043 }
|
|
|
1044
|
|
|
1045 /* From AutoFDO profiles, find values inside STMT for that we want to measure
|
|
|
1046 histograms and adds them to list VALUES. */
|
|
|
1047
|
|
|
1048 static void
|
|
|
1049 afdo_vpt (gimple_stmt_iterator *gsi, const icall_target_map &map,
|
|
|
1050 bool transform)
|
|
|
1051 {
|
|
|
1052 afdo_indirect_call (gsi, map, transform);
|
|
|
1053 }
|
|
|
1054
|
|
|
1055 typedef std::set<basic_block> bb_set;
|
|
|
1056 typedef std::set<edge> edge_set;
|
|
|
1057
|
|
|
1058 static bool
|
|
|
1059 is_bb_annotated (const basic_block bb, const bb_set &annotated)
|
|
|
1060 {
|
|
|
1061 return annotated.find (bb) != annotated.end ();
|
|
|
1062 }
|
|
|
1063
|
|
|
1064 static void
|
|
|
1065 set_bb_annotated (basic_block bb, bb_set *annotated)
|
|
|
1066 {
|
|
|
1067 annotated->insert (bb);
|
|
|
1068 }
|
|
|
1069
|
|
|
1070 static bool
|
|
|
1071 is_edge_annotated (const edge e, const edge_set &annotated)
|
|
|
1072 {
|
|
|
1073 return annotated.find (e) != annotated.end ();
|
|
|
1074 }
|
|
|
1075
|
|
|
1076 static void
|
|
|
1077 set_edge_annotated (edge e, edge_set *annotated)
|
|
|
1078 {
|
|
|
1079 annotated->insert (e);
|
|
|
1080 }
|
|
|
1081
|
|
|
1082 /* For a given BB, set its execution count. Attach value profile if a stmt
|
|
|
1083 is not in PROMOTED, because we only want to promote an indirect call once.
|
|
|
1084 Return TRUE if BB is annotated. */
|
|
|
1085
|
|
|
1086 static bool
|
|
|
1087 afdo_set_bb_count (basic_block bb, const stmt_set &promoted)
|
|
|
1088 {
|
|
|
1089 gimple_stmt_iterator gsi;
|
|
|
1090 edge e;
|
|
|
1091 edge_iterator ei;
|
|
|
1092 gcov_type max_count = 0;
|
|
|
1093 bool has_annotated = false;
|
|
|
1094
|
|
|
1095 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
1096 {
|
|
|
1097 count_info info;
|
|
|
1098 gimple *stmt = gsi_stmt (gsi);
|
|
|
1099 if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
|
|
|
1100 continue;
|
|
|
1101 if (afdo_source_profile->get_count_info (stmt, &info))
|
|
|
1102 {
|
|
|
1103 if (info.count > max_count)
|
|
|
1104 max_count = info.count;
|
|
|
1105 has_annotated = true;
|
|
|
1106 if (info.targets.size () > 0
|
|
|
1107 && promoted.find (stmt) == promoted.end ())
|
|
|
1108 afdo_vpt (&gsi, info.targets, false);
|
|
|
1109 }
|
|
|
1110 }
|
|
|
1111
|
|
|
1112 if (!has_annotated)
|
|
|
1113 return false;
|
|
|
1114
|
|
|
1115 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
1116 afdo_source_profile->mark_annotated (gimple_location (gsi_stmt (gsi)));
|
|
|
1117 for (gphi_iterator gpi = gsi_start_phis (bb);
|
|
|
1118 !gsi_end_p (gpi);
|
|
|
1119 gsi_next (&gpi))
|
|
|
1120 {
|
|
|
1121 gphi *phi = gpi.phi ();
|
|
|
1122 size_t i;
|
|
|
1123 for (i = 0; i < gimple_phi_num_args (phi); i++)
|
|
|
1124 afdo_source_profile->mark_annotated (gimple_phi_arg_location (phi, i));
|
|
|
1125 }
|
|
|
1126 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
1127 afdo_source_profile->mark_annotated (e->goto_locus);
|
|
|
1128
|
|
|
1129 bb->count = profile_count::from_gcov_type (max_count).afdo ();
|
|
|
1130 return true;
|
|
|
1131 }
|
|
|
1132
|
|
|
1133 /* BB1 and BB2 are in an equivalent class iff:
|
|
|
1134 1. BB1 dominates BB2.
|
|
|
1135 2. BB2 post-dominates BB1.
|
|
|
1136 3. BB1 and BB2 are in the same loop nest.
|
|
|
1137 This function finds the equivalent class for each basic block, and
|
|
|
1138 stores a pointer to the first BB in its equivalent class. Meanwhile,
|
|
|
1139 set bb counts for the same equivalent class to be idenical. Update
|
|
|
1140 ANNOTATED_BB for the first BB in its equivalent class. */
|
|
|
1141
|
|
|
1142 static void
|
|
|
1143 afdo_find_equiv_class (bb_set *annotated_bb)
|
|
|
1144 {
|
|
|
1145 basic_block bb;
|
|
|
1146
|
|
|
1147 FOR_ALL_BB_FN (bb, cfun)
|
|
|
1148 bb->aux = NULL;
|
|
|
1149
|
|
|
1150 FOR_ALL_BB_FN (bb, cfun)
|
|
|
1151 {
|
|
|
1152 vec<basic_block> dom_bbs;
|
|
|
1153 basic_block bb1;
|
|
|
1154 int i;
|
|
|
1155
|
|
|
1156 if (bb->aux != NULL)
|
|
|
1157 continue;
|
|
|
1158 bb->aux = bb;
|
|
|
1159 dom_bbs = get_dominated_by (CDI_DOMINATORS, bb);
|
|
|
1160 FOR_EACH_VEC_ELT (dom_bbs, i, bb1)
|
|
|
1161 if (bb1->aux == NULL && dominated_by_p (CDI_POST_DOMINATORS, bb, bb1)
|
|
|
1162 && bb1->loop_father == bb->loop_father)
|
|
|
1163 {
|
|
|
1164 bb1->aux = bb;
|
|
|
1165 if (bb1->count > bb->count && is_bb_annotated (bb1, *annotated_bb))
|
|
|
1166 {
|
|
|
1167 bb->count = bb1->count;
|
|
|
1168 set_bb_annotated (bb, annotated_bb);
|
|
|
1169 }
|
|
|
1170 }
|
|
|
1171 dom_bbs = get_dominated_by (CDI_POST_DOMINATORS, bb);
|
|
|
1172 FOR_EACH_VEC_ELT (dom_bbs, i, bb1)
|
|
|
1173 if (bb1->aux == NULL && dominated_by_p (CDI_DOMINATORS, bb, bb1)
|
|
|
1174 && bb1->loop_father == bb->loop_father)
|
|
|
1175 {
|
|
|
1176 bb1->aux = bb;
|
|
|
1177 if (bb1->count > bb->count && is_bb_annotated (bb1, *annotated_bb))
|
|
|
1178 {
|
|
|
1179 bb->count = bb1->count;
|
|
|
1180 set_bb_annotated (bb, annotated_bb);
|
|
|
1181 }
|
|
|
1182 }
|
|
|
1183 }
|
|
|
1184 }
|
|
|
1185
|
|
|
1186 /* If a basic block's count is known, and only one of its in/out edges' count
|
|
|
1187 is unknown, its count can be calculated. Meanwhile, if all of the in/out
|
|
|
1188 edges' counts are known, then the basic block's unknown count can also be
|
|
|
1189 calculated.
|
|
|
1190 IS_SUCC is true if out edges of a basic blocks are examined.
|
|
|
1191 Update ANNOTATED_BB and ANNOTATED_EDGE accordingly.
|
|
|
1192 Return TRUE if any basic block/edge count is changed. */
|
|
|
1193
|
|
|
1194 static bool
|
|
|
1195 afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
|
|
|
1196 edge_set *annotated_edge)
|
|
|
1197 {
|
|
|
1198 basic_block bb;
|
|
|
1199 bool changed = false;
|
|
|
1200
|
|
|
1201 FOR_EACH_BB_FN (bb, cfun)
|
|
|
1202 {
|
|
|
1203 edge e, unknown_edge = NULL;
|
|
|
1204 edge_iterator ei;
|
|
|
1205 int num_unknown_edge = 0;
|
|
|
1206 profile_count total_known_count = profile_count::zero ().afdo ();
|
|
|
1207
|
|
|
1208 FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
|
|
|
1209 if (!is_edge_annotated (e, *annotated_edge))
|
|
|
1210 num_unknown_edge++, unknown_edge = e;
|
|
|
1211 else
|
|
|
1212 total_known_count += e->count ();
|
|
|
1213
|
|
|
1214 if (num_unknown_edge == 0)
|
|
|
1215 {
|
|
|
1216 if (total_known_count > bb->count)
|
|
|
1217 {
|
|
|
1218 bb->count = total_known_count;
|
|
|
1219 changed = true;
|
|
|
1220 }
|
|
|
1221 if (!is_bb_annotated (bb, *annotated_bb))
|
|
|
1222 {
|
|
|
1223 set_bb_annotated (bb, annotated_bb);
|
|
|
1224 changed = true;
|
|
|
1225 }
|
|
|
1226 }
|
|
|
1227 else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
|
|
|
1228 {
|
|
|
1229 unknown_edge->probability
|
|
|
1230 = total_known_count.probability_in (bb->count);
|
|
|
1231 set_edge_annotated (unknown_edge, annotated_edge);
|
|
|
1232 changed = true;
|
|
|
1233 }
|
|
|
1234 }
|
|
|
1235 return changed;
|
|
|
1236 }
|
|
|
1237
|
|
|
1238 /* Special propagation for circuit expressions. Because GCC translates
|
|
|
1239 control flow into data flow for circuit expressions. E.g.
|
|
|
1240 BB1:
|
|
|
1241 if (a && b)
|
|
|
1242 BB2
|
|
|
1243 else
|
|
|
1244 BB3
|
|
|
1245
|
|
|
1246 will be translated into:
|
|
|
1247
|
|
|
1248 BB1:
|
|
|
1249 if (a)
|
|
|
1250 goto BB.t1
|
|
|
1251 else
|
|
|
1252 goto BB.t3
|
|
|
1253 BB.t1:
|
|
|
1254 if (b)
|
|
|
1255 goto BB.t2
|
|
|
1256 else
|
|
|
1257 goto BB.t3
|
|
|
1258 BB.t2:
|
|
|
1259 goto BB.t3
|
|
|
1260 BB.t3:
|
|
|
1261 tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
|
|
|
1262 if (tmp)
|
|
|
1263 goto BB2
|
|
|
1264 else
|
|
|
1265 goto BB3
|
|
|
1266
|
|
|
1267 In this case, we need to propagate through PHI to determine the edge
|
|
|
1268 count of BB1->BB.t1, BB.t1->BB.t2.
|
|
|
1269 Update ANNOTATED_EDGE accordingly. */
|
|
|
1270
|
|
|
1271 static void
|
|
|
1272 afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
|
|
|
1273 {
|
|
|
1274 basic_block bb;
|
|
|
1275 FOR_ALL_BB_FN (bb, cfun)
|
|
|
1276 {
|
|
|
1277 gimple *def_stmt;
|
|
|
1278 tree cmp_rhs, cmp_lhs;
|
|
|
1279 gimple *cmp_stmt = last_stmt (bb);
|
|
|
1280 edge e;
|
|
|
1281 edge_iterator ei;
|
|
|
1282
|
|
|
1283 if (!cmp_stmt || gimple_code (cmp_stmt) != GIMPLE_COND)
|
|
|
1284 continue;
|
|
|
1285 cmp_rhs = gimple_cond_rhs (cmp_stmt);
|
|
|
1286 cmp_lhs = gimple_cond_lhs (cmp_stmt);
|
|
|
1287 if (!TREE_CONSTANT (cmp_rhs)
|
|
|
1288 || !(integer_zerop (cmp_rhs) || integer_onep (cmp_rhs)))
|
|
|
1289 continue;
|
|
|
1290 if (TREE_CODE (cmp_lhs) != SSA_NAME)
|
|
|
1291 continue;
|
|
|
1292 if (!is_bb_annotated (bb, annotated_bb))
|
|
|
1293 continue;
|
|
|
1294 def_stmt = SSA_NAME_DEF_STMT (cmp_lhs);
|
|
|
1295 while (def_stmt && gimple_code (def_stmt) == GIMPLE_ASSIGN
|
|
|
1296 && gimple_assign_single_p (def_stmt)
|
|
|
1297 && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
|
|
|
1298 def_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
|
|
|
1299 if (!def_stmt)
|
|
|
1300 continue;
|
|
|
1301 gphi *phi_stmt = dyn_cast <gphi *> (def_stmt);
|
|
|
1302 if (!phi_stmt)
|
|
|
1303 continue;
|
|
|
1304 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
1305 {
|
|
|
1306 unsigned i, total = 0;
|
|
|
1307 edge only_one;
|
|
|
1308 bool check_value_one = (((integer_onep (cmp_rhs))
|
|
|
1309 ^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
|
|
|
1310 ^ ((e->flags & EDGE_TRUE_VALUE) != 0));
|
|
|
1311 if (!is_edge_annotated (e, *annotated_edge))
|
|
|
1312 continue;
|
|
|
1313 for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
|
|
|
1314 {
|
|
|
1315 tree val = gimple_phi_arg_def (phi_stmt, i);
|
|
|
1316 edge ep = gimple_phi_arg_edge (phi_stmt, i);
|
|
|
1317
|
|
|
1318 if (!TREE_CONSTANT (val)
|
|
|
1319 || !(integer_zerop (val) || integer_onep (val)))
|
|
|
1320 continue;
|
|
|
1321 if (check_value_one ^ integer_onep (val))
|
|
|
1322 continue;
|
|
|
1323 total++;
|
|
|
1324 only_one = ep;
|
|
|
1325 if (!e->probability.initialized_p ()
|
|
|
1326 && !is_edge_annotated (ep, *annotated_edge))
|
|
|
1327 {
|
|
|
1328 ep->probability = profile_probability::never ().afdo ();
|
|
|
1329 set_edge_annotated (ep, annotated_edge);
|
|
|
1330 }
|
|
|
1331 }
|
|
|
1332 if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
|
|
|
1333 {
|
|
|
1334 only_one->probability = e->probability;
|
|
|
1335 set_edge_annotated (only_one, annotated_edge);
|
|
|
1336 }
|
|
|
1337 }
|
|
|
1338 }
|
|
|
1339 }
|
|
|
1340
|
|
|
1341 /* Propagate the basic block count and edge count on the control flow
|
|
|
1342 graph. We do the propagation iteratively until stablize. */
|
|
|
1343
|
|
|
1344 static void
|
|
|
1345 afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
|
|
|
1346 {
|
|
|
1347 basic_block bb;
|
|
|
1348 bool changed = true;
|
|
|
1349 int i = 0;
|
|
|
1350
|
|
|
1351 FOR_ALL_BB_FN (bb, cfun)
|
|
|
1352 {
|
|
|
1353 bb->count = ((basic_block)bb->aux)->count;
|
|
|
1354 if (is_bb_annotated ((basic_block)bb->aux, *annotated_bb))
|
|
|
1355 set_bb_annotated (bb, annotated_bb);
|
|
|
1356 }
|
|
|
1357
|
|
|
1358 while (changed && i++ < 10)
|
|
|
1359 {
|
|
|
1360 changed = false;
|
|
|
1361
|
|
|
1362 if (afdo_propagate_edge (true, annotated_bb, annotated_edge))
|
|
|
1363 changed = true;
|
|
|
1364 if (afdo_propagate_edge (false, annotated_bb, annotated_edge))
|
|
|
1365 changed = true;
|
|
|
1366 afdo_propagate_circuit (*annotated_bb, annotated_edge);
|
|
|
1367 }
|
|
|
1368 }
|
|
|
1369
|
|
|
1370 /* Propagate counts on control flow graph and calculate branch
|
|
|
1371 probabilities. */
|
|
|
1372
|
|
|
1373 static void
|
|
|
1374 afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge)
|
|
|
1375 {
|
|
|
1376 basic_block bb;
|
|
|
1377 bool has_sample = false;
|
|
|
1378
|
|
|
1379 FOR_EACH_BB_FN (bb, cfun)
|
|
|
1380 {
|
|
|
1381 if (bb->count > profile_count::zero ())
|
|
|
1382 {
|
|
|
1383 has_sample = true;
|
|
|
1384 break;
|
|
|
1385 }
|
|
|
1386 }
|
|
|
1387
|
|
|
1388 if (!has_sample)
|
|
|
1389 return;
|
|
|
1390
|
|
|
1391 calculate_dominance_info (CDI_POST_DOMINATORS);
|
|
|
1392 calculate_dominance_info (CDI_DOMINATORS);
|
|
|
1393 loop_optimizer_init (0);
|
|
|
1394
|
|
|
1395 afdo_find_equiv_class (annotated_bb);
|
|
|
1396 afdo_propagate (annotated_bb, annotated_edge);
|
|
|
1397
|
|
|
1398 FOR_EACH_BB_FN (bb, cfun)
|
|
|
1399 {
|
|
|
1400 edge e;
|
|
|
1401 edge_iterator ei;
|
|
|
1402 int num_unknown_succ = 0;
|
|
|
1403 profile_count total_count = profile_count::zero ();
|
|
|
1404
|
|
|
1405 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
1406 {
|
|
|
1407 if (!is_edge_annotated (e, *annotated_edge))
|
|
|
1408 num_unknown_succ++;
|
|
|
1409 else
|
|
|
1410 total_count += e->count ();
|
|
|
1411 }
|
|
|
1412 if (num_unknown_succ == 0 && total_count > profile_count::zero ())
|
|
|
1413 {
|
|
|
1414 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
1415 e->probability = e->count ().probability_in (total_count);
|
|
|
1416 }
|
|
|
1417 }
|
|
|
1418 FOR_ALL_BB_FN (bb, cfun)
|
|
|
1419 bb->aux = NULL;
|
|
|
1420
|
|
|
1421 loop_optimizer_finalize ();
|
|
|
1422 free_dominance_info (CDI_DOMINATORS);
|
|
|
1423 free_dominance_info (CDI_POST_DOMINATORS);
|
|
|
1424 }
|
|
|
1425
|
|
|
1426 /* Perform value profile transformation using AutoFDO profile. Add the
|
|
|
1427 promoted stmts to PROMOTED_STMTS. Return TRUE if there is any
|
|
|
1428 indirect call promoted. */
|
|
|
1429
|
|
|
1430 static bool
|
|
|
1431 afdo_vpt_for_early_inline (stmt_set *promoted_stmts)
|
|
|
1432 {
|
|
|
1433 basic_block bb;
|
|
|
1434 if (afdo_source_profile->get_function_instance_by_decl (
|
|
|
1435 current_function_decl) == NULL)
|
|
|
1436 return false;
|
|
|
1437
|
|
|
1438 compute_fn_summary (cgraph_node::get (current_function_decl), true);
|
|
|
1439
|
|
|
1440 bool has_vpt = false;
|
|
|
1441 FOR_EACH_BB_FN (bb, cfun)
|
|
|
1442 {
|
|
|
1443 if (!has_indirect_call (bb))
|
|
|
1444 continue;
|
|
|
1445 gimple_stmt_iterator gsi;
|
|
|
1446
|
|
|
1447 gcov_type bb_count = 0;
|
|
|
1448 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
1449 {
|
|
|
1450 count_info info;
|
|
|
1451 gimple *stmt = gsi_stmt (gsi);
|
|
|
1452 if (afdo_source_profile->get_count_info (stmt, &info))
|
|
|
1453 bb_count = MAX (bb_count, info.count);
|
|
|
1454 }
|
|
|
1455
|
|
|
1456 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
|
|
1457 {
|
|
|
1458 gcall *stmt = dyn_cast <gcall *> (gsi_stmt (gsi));
|
|
|
1459 /* IC_promotion and early_inline_2 is done in multiple iterations.
|
|
|
1460 No need to promoted the stmt if its in promoted_stmts (means
|
|
|
1461 it is already been promoted in the previous iterations). */
|
|
|
1462 if ((!stmt) || gimple_call_fn (stmt) == NULL
|
|
|
1463 || TREE_CODE (gimple_call_fn (stmt)) == FUNCTION_DECL
|
|
|
1464 || promoted_stmts->find (stmt) != promoted_stmts->end ())
|
|
|
1465 continue;
|
|
|
1466
|
|
|
1467 count_info info;
|
|
|
1468 afdo_source_profile->get_count_info (stmt, &info);
|
|
|
1469 info.count = bb_count;
|
|
|
1470 if (afdo_source_profile->update_inlined_ind_target (stmt, &info))
|
|
|
1471 {
|
|
|
1472 /* Promote the indirect call and update the promoted_stmts. */
|
|
|
1473 promoted_stmts->insert (stmt);
|
|
|
1474 afdo_vpt (&gsi, info.targets, true);
|
|
|
1475 has_vpt = true;
|
|
|
1476 }
|
|
|
1477 }
|
|
|
1478 }
|
|
|
1479
|
|
|
1480 if (has_vpt)
|
|
|
1481 {
|
|
|
1482 unsigned todo = optimize_inline_calls (current_function_decl);
|
|
|
1483 if (todo & TODO_update_ssa_any)
|
|
|
1484 update_ssa (TODO_update_ssa);
|
|
|
1485 return true;
|
|
|
1486 }
|
|
|
1487
|
|
|
1488 return false;
|
|
|
1489 }
|
|
|
1490
|
|
|
1491 /* Annotate auto profile to the control flow graph. Do not annotate value
|
|
|
1492 profile for stmts in PROMOTED_STMTS. */
|
|
|
1493
|
|
|
1494 static void
|
|
|
1495 afdo_annotate_cfg (const stmt_set &promoted_stmts)
|
|
|
1496 {
|
|
|
1497 basic_block bb;
|
|
|
1498 bb_set annotated_bb;
|
|
|
1499 edge_set annotated_edge;
|
|
|
1500 const function_instance *s
|
|
|
1501 = afdo_source_profile->get_function_instance_by_decl (
|
|
|
1502 current_function_decl);
|
|
|
1503
|
|
|
1504 if (s == NULL)
|
|
|
1505 return;
|
|
|
1506 cgraph_node::get (current_function_decl)->count
|
|
|
1507 = profile_count::from_gcov_type (s->head_count ()).afdo ();
|
|
|
1508 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
|
|
|
1509 = profile_count::from_gcov_type (s->head_count ()).afdo ();
|
|
|
1510 profile_count max_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
|
1511
|
|
|
1512 FOR_EACH_BB_FN (bb, cfun)
|
|
|
1513 {
|
|
|
1514 edge e;
|
|
|
1515 edge_iterator ei;
|
|
|
1516
|
|
|
1517 /* As autoFDO uses sampling approach, we have to assume that all
|
|
|
1518 counters are zero when not seen by autoFDO. */
|
|
|
1519 bb->count = profile_count::zero ().afdo ();
|
|
|
1520 FOR_EACH_EDGE (e, ei, bb->succs)
|
|
|
1521 e->probability = profile_probability::uninitialized ();
|
|
|
1522
|
|
|
1523 if (afdo_set_bb_count (bb, promoted_stmts))
|
|
|
1524 set_bb_annotated (bb, &annotated_bb);
|
|
|
1525 if (bb->count > max_count)
|
|
|
1526 max_count = bb->count;
|
|
|
1527 }
|
|
|
1528 if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
|
|
|
1529 > ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count)
|
|
|
1530 {
|
|
|
1531 ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count
|
|
|
1532 = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
|
1533 set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb, &annotated_bb);
|
|
|
1534 }
|
|
|
1535 if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
|
|
|
1536 > EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count)
|
|
|
1537 {
|
|
|
1538 EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count
|
|
|
1539 = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
|
|
|
1540 set_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb, &annotated_bb);
|
|
|
1541 }
|
|
|
1542 afdo_source_profile->mark_annotated (
|
|
|
1543 DECL_SOURCE_LOCATION (current_function_decl));
|
|
|
1544 afdo_source_profile->mark_annotated (cfun->function_start_locus);
|
|
|
1545 afdo_source_profile->mark_annotated (cfun->function_end_locus);
|
|
|
1546 if (max_count > profile_count::zero ())
|
|
|
1547 {
|
|
|
1548 afdo_calculate_branch_prob (&annotated_bb, &annotated_edge);
|
|
131
|
1549 update_max_bb_count ();
|
|
111
|
1550 profile_status_for_fn (cfun) = PROFILE_READ;
|
|
|
1551 }
|
|
|
1552 if (flag_value_profile_transformations)
|
|
|
1553 {
|
|
|
1554 gimple_value_profile_transformations ();
|
|
|
1555 free_dominance_info (CDI_DOMINATORS);
|
|
|
1556 free_dominance_info (CDI_POST_DOMINATORS);
|
|
|
1557 update_ssa (TODO_update_ssa);
|
|
|
1558 }
|
|
|
1559 }
|
|
|
1560
|
|
|
1561 /* Wrapper function to invoke early inliner. */
|
|
|
1562
|
|
|
1563 static void
|
|
|
1564 early_inline ()
|
|
|
1565 {
|
|
|
1566 compute_fn_summary (cgraph_node::get (current_function_decl), true);
|
|
|
1567 unsigned todo = early_inliner (cfun);
|
|
|
1568 if (todo & TODO_update_ssa_any)
|
|
|
1569 update_ssa (TODO_update_ssa);
|
|
|
1570 }
|
|
|
1571
|
|
|
1572 /* Use AutoFDO profile to annoate the control flow graph.
|
|
|
1573 Return the todo flag. */
|
|
|
1574
|
|
|
1575 static unsigned int
|
|
|
1576 auto_profile (void)
|
|
|
1577 {
|
|
|
1578 struct cgraph_node *node;
|
|
|
1579
|
|
|
1580 if (symtab->state == FINISHED)
|
|
|
1581 return 0;
|
|
|
1582
|
|
|
1583 init_node_map (true);
|
|
|
1584 profile_info = autofdo::afdo_profile_info;
|
|
|
1585
|
|
|
1586 FOR_EACH_FUNCTION (node)
|
|
|
1587 {
|
|
|
1588 if (!gimple_has_body_p (node->decl))
|
|
|
1589 continue;
|
|
|
1590
|
|
|
1591 /* Don't profile functions produced for builtin stuff. */
|
|
|
1592 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
|
|
|
1593 continue;
|
|
|
1594
|
|
|
1595 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
|
|
|
1596
|
|
|
1597 /* First do indirect call promotion and early inline to make the
|
|
|
1598 IR match the profiled binary before actual annotation.
|
|
|
1599
|
|
|
1600 This is needed because an indirect call might have been promoted
|
|
|
1601 and inlined in the profiled binary. If we do not promote and
|
|
|
1602 inline these indirect calls before annotation, the profile for
|
|
|
1603 these promoted functions will be lost.
|
|
|
1604
|
|
|
1605 e.g. foo() --indirect_call--> bar()
|
|
|
1606 In profiled binary, the callsite is promoted and inlined, making
|
|
|
1607 the profile look like:
|
|
|
1608
|
|
|
1609 foo: {
|
|
|
1610 loc_foo_1: count_1
|
|
|
1611 bar@loc_foo_2: {
|
|
|
1612 loc_bar_1: count_2
|
|
|
1613 loc_bar_2: count_3
|
|
|
1614 }
|
|
|
1615 }
|
|
|
1616
|
|
|
1617 Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
|
|
|
1618 If we perform annotation on it, the profile inside bar@loc_foo2
|
|
|
1619 will be wasted.
|
|
|
1620
|
|
|
1621 To avoid this, we promote loc_foo_2 and inline the promoted bar
|
|
|
1622 function before annotation, so the profile inside bar@loc_foo2
|
|
|
1623 will be useful. */
|
|
|
1624 autofdo::stmt_set promoted_stmts;
|
|
|
1625 for (int i = 0; i < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS); i++)
|
|
|
1626 {
|
|
|
1627 if (!flag_value_profile_transformations
|
|
|
1628 || !autofdo::afdo_vpt_for_early_inline (&promoted_stmts))
|
|
|
1629 break;
|
|
|
1630 early_inline ();
|
|
|
1631 }
|
|
|
1632
|
|
|
1633 early_inline ();
|
|
|
1634 autofdo::afdo_annotate_cfg (promoted_stmts);
|
|
|
1635 compute_function_frequency ();
|
|
|
1636
|
|
|
1637 /* Local pure-const may imply need to fixup the cfg. */
|
|
|
1638 if (execute_fixup_cfg () & TODO_cleanup_cfg)
|
|
|
1639 cleanup_tree_cfg ();
|
|
|
1640
|
|
|
1641 free_dominance_info (CDI_DOMINATORS);
|
|
|
1642 free_dominance_info (CDI_POST_DOMINATORS);
|
|
|
1643 cgraph_edge::rebuild_edges ();
|
|
|
1644 compute_fn_summary (cgraph_node::get (current_function_decl), true);
|
|
|
1645 pop_cfun ();
|
|
|
1646 }
|
|
|
1647
|
|
|
1648 return TODO_rebuild_cgraph_edges;
|
|
|
1649 }
|
|
|
1650 } /* namespace autofdo. */
|
|
|
1651
|
|
|
1652 /* Read the profile from the profile data file. */
|
|
|
1653
|
|
|
1654 void
|
|
|
1655 read_autofdo_file (void)
|
|
|
1656 {
|
|
|
1657 if (auto_profile_file == NULL)
|
|
|
1658 auto_profile_file = DEFAULT_AUTO_PROFILE_FILE;
|
|
|
1659
|
|
131
|
1660 autofdo::afdo_profile_info = XNEW (gcov_summary);
|
|
111
|
1661 autofdo::afdo_profile_info->runs = 1;
|
|
|
1662 autofdo::afdo_profile_info->sum_max = 0;
|
|
|
1663
|
|
|
1664 /* Read the profile from the profile file. */
|
|
|
1665 autofdo::read_profile ();
|
|
|
1666 }
|
|
|
1667
|
|
|
1668 /* Free the resources. */
|
|
|
1669
|
|
|
1670 void
|
|
|
1671 end_auto_profile (void)
|
|
|
1672 {
|
|
|
1673 delete autofdo::afdo_source_profile;
|
|
|
1674 delete autofdo::afdo_string_table;
|
|
|
1675 profile_info = NULL;
|
|
|
1676 }
|
|
|
1677
|
|
|
1678 /* Returns TRUE if EDGE is hot enough to be inlined early. */
|
|
|
1679
|
|
|
1680 bool
|
|
|
1681 afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge)
|
|
|
1682 {
|
|
|
1683 gcov_type count
|
|
|
1684 = autofdo::afdo_source_profile->get_callsite_total_count (edge);
|
|
|
1685
|
|
|
1686 if (count > 0)
|
|
|
1687 {
|
|
|
1688 bool is_hot;
|
|
131
|
1689 gcov_summary *saved_profile_info = profile_info;
|
|
111
|
1690 /* At early inline stage, profile_info is not set yet. We need to
|
|
|
1691 temporarily set it to afdo_profile_info to calculate hotness. */
|
|
|
1692 profile_info = autofdo::afdo_profile_info;
|
|
|
1693 is_hot = maybe_hot_count_p (NULL, profile_count::from_gcov_type (count));
|
|
|
1694 profile_info = saved_profile_info;
|
|
|
1695 return is_hot;
|
|
|
1696 }
|
|
|
1697
|
|
|
1698 return false;
|
|
|
1699 }
|
|
|
1700
|
|
|
1701 namespace
|
|
|
1702 {
|
|
|
1703
|
|
|
1704 const pass_data pass_data_ipa_auto_profile = {
|
|
|
1705 SIMPLE_IPA_PASS, "afdo", /* name */
|
|
|
1706 OPTGROUP_NONE, /* optinfo_flags */
|
|
|
1707 TV_IPA_AUTOFDO, /* tv_id */
|
|
|
1708 0, /* properties_required */
|
|
|
1709 0, /* properties_provided */
|
|
|
1710 0, /* properties_destroyed */
|
|
|
1711 0, /* todo_flags_start */
|
|
|
1712 0, /* todo_flags_finish */
|
|
|
1713 };
|
|
|
1714
|
|
|
1715 class pass_ipa_auto_profile : public simple_ipa_opt_pass
|
|
|
1716 {
|
|
|
1717 public:
|
|
|
1718 pass_ipa_auto_profile (gcc::context *ctxt)
|
|
|
1719 : simple_ipa_opt_pass (pass_data_ipa_auto_profile, ctxt)
|
|
|
1720 {
|
|
|
1721 }
|
|
|
1722
|
|
|
1723 /* opt_pass methods: */
|
|
|
1724 virtual bool
|
|
|
1725 gate (function *)
|
|
|
1726 {
|
|
|
1727 return flag_auto_profile;
|
|
|
1728 }
|
|
|
1729 virtual unsigned int
|
|
|
1730 execute (function *)
|
|
|
1731 {
|
|
|
1732 return autofdo::auto_profile ();
|
|
|
1733 }
|
|
|
1734 }; // class pass_ipa_auto_profile
|
|
|
1735
|
|
|
1736 } // anon namespace
|
|
|
1737
|
|
|
1738 simple_ipa_opt_pass *
|
|
|
1739 make_pass_ipa_auto_profile (gcc::context *ctxt)
|
|
|
1740 {
|
|
|
1741 return new pass_ipa_auto_profile (ctxt);
|
|
|
1742 }
|
