Mercurial > hg > CbC > CbC_gcc
comparison gcc/profile-count.h @ 132:d34655255c78
update gcc-8.2
author | mir3636 |
---|---|
date | Thu, 25 Oct 2018 10:21:07 +0900 |
parents | 84e7813d76e9 |
children | 1830386684a0 |
comparison
equal
deleted
inserted
replaced
130:e108057fa461 | 132:d34655255c78 |
---|---|
1 /* Profile counter container type. | 1 /* Profile counter container type. |
2 Copyright (C) 2017 Free Software Foundation, Inc. | 2 Copyright (C) 2017-2018 Free Software Foundation, Inc. |
3 Contributed by Jan Hubicka | 3 Contributed by Jan Hubicka |
4 | 4 |
5 This file is part of GCC. | 5 This file is part of GCC. |
6 | 6 |
7 GCC is free software; you can redistribute it and/or modify it under | 7 GCC is free software; you can redistribute it and/or modify it under |
19 <http://www.gnu.org/licenses/>. */ | 19 <http://www.gnu.org/licenses/>. */ |
20 | 20 |
21 #ifndef GCC_PROFILE_COUNT_H | 21 #ifndef GCC_PROFILE_COUNT_H |
22 #define GCC_PROFILE_COUNT_H | 22 #define GCC_PROFILE_COUNT_H |
23 | 23 |
24 struct function; | |
25 class profile_count; | |
26 | |
24 /* Quality of the profile count. Because gengtype does not support enums | 27 /* Quality of the profile count. Because gengtype does not support enums |
25 inside of classes, this is in global namespace. */ | 28 inside of classes, this is in global namespace. */ |
26 enum profile_quality { | 29 enum profile_quality { |
30 /* Uninitialized value. */ | |
31 profile_uninitialized, | |
32 /* Profile is based on static branch prediction heuristics and may | |
33 or may not match reality. It is local to function and can not be compared | |
34 inter-procedurally. Never used by probabilities (they are always local). | |
35 */ | |
36 profile_guessed_local, | |
37 /* Profile was read by feedback and was 0, we used local heuristics to guess | |
38 better. This is the case of functions not run in profile fedback. | |
39 Never used by probabilities. */ | |
40 profile_guessed_global0, | |
41 | |
42 /* Same as profile_guessed_global0 but global count is adjusted 0. */ | |
43 profile_guessed_global0adjusted, | |
44 | |
27 /* Profile is based on static branch prediction heuristics. It may or may | 45 /* Profile is based on static branch prediction heuristics. It may or may |
28 not reflect the reality. */ | 46 not reflect the reality but it can be compared interprocedurally |
29 profile_guessed = 0, | 47 (for example, we inlined function w/o profile feedback into function |
48 with feedback and propagated from that). | |
49 Never used by probablities. */ | |
50 profile_guessed, | |
30 /* Profile was determined by autofdo. */ | 51 /* Profile was determined by autofdo. */ |
31 profile_afdo = 1, | 52 profile_afdo, |
32 /* Profile was originally based on feedback but it was adjusted | 53 /* Profile was originally based on feedback but it was adjusted |
33 by code duplicating optimization. It may not precisely reflect the | 54 by code duplicating optimization. It may not precisely reflect the |
34 particular code path. */ | 55 particular code path. */ |
35 profile_adjusted = 2, | 56 profile_adjusted, |
36 /* Profile was read from profile feedback or determined by accurate static | 57 /* Profile was read from profile feedback or determined by accurate static |
37 method. */ | 58 method. */ |
38 profile_precise = 3 | 59 profile_precise |
39 }; | 60 }; |
61 | |
62 extern const char *profile_quality_as_string (enum profile_quality); | |
40 | 63 |
41 /* The base value for branch probability notes and edge probabilities. */ | 64 /* The base value for branch probability notes and edge probabilities. */ |
42 #define REG_BR_PROB_BASE 10000 | 65 #define REG_BR_PROB_BASE 10000 |
43 | 66 |
44 #define RDIV(X,Y) (((X) + (Y) / 2) / (Y)) | 67 #define RDIV(X,Y) (((X) + (Y) / 2) / (Y)) |
112 these conversions will probably go away because they are lossy. | 135 these conversions will probably go away because they are lossy. |
113 */ | 136 */ |
114 | 137 |
115 class GTY((user)) profile_probability | 138 class GTY((user)) profile_probability |
116 { | 139 { |
117 static const int n_bits = 30; | 140 static const int n_bits = 29; |
118 /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that | 141 /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that |
119 will lead to harder multiplication sequences. */ | 142 will lead to harder multiplication sequences. */ |
120 static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2); | 143 static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2); |
121 static const uint32_t uninitialized_probability | 144 static const uint32_t uninitialized_probability |
122 = ((uint32_t) 1 << (n_bits - 1)) - 1; | 145 = ((uint32_t) 1 << (n_bits - 1)) - 1; |
123 | 146 |
124 uint32_t m_val : 30; | 147 uint32_t m_val : 29; |
125 enum profile_quality m_quality : 2; | 148 enum profile_quality m_quality : 3; |
126 | 149 |
127 friend class profile_count; | 150 friend class profile_count; |
128 public: | 151 public: |
129 | 152 |
130 /* Named probabilities. */ | 153 /* Named probabilities. */ |
144 } | 167 } |
145 static profile_probability very_unlikely () | 168 static profile_probability very_unlikely () |
146 { | 169 { |
147 /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */ | 170 /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */ |
148 profile_probability r | 171 profile_probability r |
149 = profile_probability::always ().apply_scale (1, 2000); | 172 = profile_probability::guessed_always ().apply_scale (1, 2000); |
150 r.m_val--; | 173 r.m_val--; |
151 return r; | 174 return r; |
152 } | 175 } |
153 static profile_probability unlikely () | 176 static profile_probability unlikely () |
154 { | 177 { |
155 /* Be consistent with PROB_VERY_LIKELY in predict.h. */ | 178 /* Be consistent with PROB_VERY_LIKELY in predict.h. */ |
156 profile_probability r | 179 profile_probability r |
157 = profile_probability::always ().apply_scale (1, 5); | 180 = profile_probability::guessed_always ().apply_scale (1, 5); |
158 r.m_val--; | 181 r.m_val--; |
159 return r; | 182 return r; |
160 } | 183 } |
161 static profile_probability even () | 184 static profile_probability even () |
162 { | 185 { |
163 return profile_probability::always ().apply_scale (1, 2); | 186 return profile_probability::guessed_always ().apply_scale (1, 2); |
164 } | 187 } |
165 static profile_probability very_likely () | 188 static profile_probability very_likely () |
166 { | 189 { |
167 return profile_probability::always () - very_unlikely (); | 190 return profile_probability::always () - very_unlikely (); |
168 } | 191 } |
224 | 247 |
225 /* Conversion to and from RTL representation of profile probabilities. */ | 248 /* Conversion to and from RTL representation of profile probabilities. */ |
226 static profile_probability from_reg_br_prob_note (int v) | 249 static profile_probability from_reg_br_prob_note (int v) |
227 { | 250 { |
228 profile_probability ret; | 251 profile_probability ret; |
229 ret.m_val = ((unsigned int)v) / 4; | 252 ret.m_val = ((unsigned int)v) / 8; |
230 ret.m_quality = (enum profile_quality)(v & 3); | 253 ret.m_quality = (enum profile_quality)(v & 7); |
231 return ret; | 254 return ret; |
232 } | 255 } |
233 int to_reg_br_prob_note () const | 256 int to_reg_br_prob_note () const |
234 { | 257 { |
235 gcc_checking_assert (initialized_p ()); | 258 gcc_checking_assert (initialized_p ()); |
236 int ret = m_val * 4 + m_quality; | 259 int ret = m_val * 8 + m_quality; |
237 gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret) | 260 gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret) |
238 == *this); | 261 == *this); |
239 return ret; | 262 return ret; |
240 } | 263 } |
241 | 264 |
328 return profile_probability::never (); | 351 return profile_probability::never (); |
329 if (!initialized_p () || !other.initialized_p ()) | 352 if (!initialized_p () || !other.initialized_p ()) |
330 return profile_probability::uninitialized (); | 353 return profile_probability::uninitialized (); |
331 profile_probability ret; | 354 profile_probability ret; |
332 ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); | 355 ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); |
333 ret.m_quality = MIN (m_quality, other.m_quality); | 356 ret.m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted); |
334 return ret; | 357 return ret; |
335 } | 358 } |
336 profile_probability &operator*= (const profile_probability &other) | 359 profile_probability &operator*= (const profile_probability &other) |
337 { | 360 { |
338 if (*this == profile_probability::never () | 361 if (*this == profile_probability::never () |
341 if (!initialized_p () || !other.initialized_p ()) | 364 if (!initialized_p () || !other.initialized_p ()) |
342 return *this = profile_probability::uninitialized (); | 365 return *this = profile_probability::uninitialized (); |
343 else | 366 else |
344 { | 367 { |
345 m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); | 368 m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); |
346 m_quality = MIN (m_quality, other.m_quality); | 369 m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted); |
347 } | 370 } |
348 return *this; | 371 return *this; |
349 } | 372 } |
350 profile_probability operator/ (const profile_probability &other) const | 373 profile_probability operator/ (const profile_probability &other) const |
351 { | 374 { |
352 if (*this == profile_probability::never ()) | 375 if (*this == profile_probability::never ()) |
353 return profile_probability::never (); | 376 return profile_probability::never (); |
354 if (!initialized_p () || !other.initialized_p ()) | 377 if (!initialized_p () || !other.initialized_p ()) |
355 return profile_probability::uninitialized (); | 378 return profile_probability::uninitialized (); |
356 profile_probability ret; | 379 profile_probability ret; |
380 /* If we get probability above 1, mark it as unreliable and return 1. */ | |
357 if (m_val >= other.m_val) | 381 if (m_val >= other.m_val) |
358 ret.m_val = max_probability; | 382 { |
383 ret.m_val = max_probability; | |
384 ret.m_quality = MIN (MIN (m_quality, other.m_quality), | |
385 profile_guessed); | |
386 return ret; | |
387 } | |
359 else if (!m_val) | 388 else if (!m_val) |
360 ret.m_val = 0; | 389 ret.m_val = 0; |
361 else | 390 else |
362 { | 391 { |
363 gcc_checking_assert (other.m_val); | 392 gcc_checking_assert (other.m_val); |
364 ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability, | 393 ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability, |
365 other.m_val), | 394 other.m_val), |
366 max_probability); | 395 max_probability); |
367 } | 396 } |
368 ret.m_quality = MIN (m_quality, other.m_quality); | 397 ret.m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted); |
369 return ret; | 398 return ret; |
370 } | 399 } |
371 profile_probability &operator/= (const profile_probability &other) | 400 profile_probability &operator/= (const profile_probability &other) |
372 { | 401 { |
373 if (*this == profile_probability::never ()) | 402 if (*this == profile_probability::never ()) |
374 return *this = profile_probability::never (); | 403 return *this = profile_probability::never (); |
375 if (!initialized_p () || !other.initialized_p ()) | 404 if (!initialized_p () || !other.initialized_p ()) |
376 return *this = profile_probability::uninitialized (); | 405 return *this = profile_probability::uninitialized (); |
377 else | 406 else |
378 { | 407 { |
408 /* If we get probability above 1, mark it as unreliable | |
409 and return 1. */ | |
379 if (m_val > other.m_val) | 410 if (m_val > other.m_val) |
380 m_val = max_probability; | 411 { |
412 m_val = max_probability; | |
413 m_quality = MIN (MIN (m_quality, other.m_quality), | |
414 profile_guessed); | |
415 return *this; | |
416 } | |
381 else if (!m_val) | 417 else if (!m_val) |
382 ; | 418 ; |
383 else | 419 else |
384 { | 420 { |
385 gcc_checking_assert (other.m_val); | 421 gcc_checking_assert (other.m_val); |
386 m_val = MIN (RDIV ((uint64_t)m_val * max_probability, | 422 m_val = MIN (RDIV ((uint64_t)m_val * max_probability, |
387 other.m_val), | 423 other.m_val), |
388 max_probability); | 424 max_probability); |
389 } | 425 } |
390 m_quality = MIN (m_quality, other.m_quality); | 426 m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted); |
391 } | 427 } |
392 return *this; | 428 return *this; |
429 } | |
430 | |
431 /* Split *THIS (ORIG) probability into 2 probabilities, such that | |
432 the returned one (FIRST) is *THIS * CPROB and *THIS is | |
433 adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND | |
434 == ORIG. This is useful e.g. when splitting a conditional | |
435 branch like: | |
436 if (cond) | |
437 goto lab; // ORIG probability | |
438 into | |
439 if (cond1) | |
440 goto lab; // FIRST = ORIG * CPROB probability | |
441 if (cond2) | |
442 goto lab; // SECOND probability | |
443 such that the overall probability of jumping to lab remains | |
444 the same. CPROB gives the relative probability between the | |
445 branches. */ | |
446 profile_probability split (const profile_probability &cprob) | |
447 { | |
448 profile_probability ret = *this * cprob; | |
449 /* The following is equivalent to: | |
450 *this = cprob.invert () * *this / ret.invert (); */ | |
451 *this = (*this - ret) / ret.invert (); | |
452 return ret; | |
393 } | 453 } |
394 | 454 |
395 gcov_type apply (gcov_type val) const | 455 gcov_type apply (gcov_type val) const |
396 { | 456 { |
397 if (*this == profile_probability::uninitialized ()) | 457 if (*this == profile_probability::uninitialized ()) |
416 /* Return THIS with quality dropped to AFDO. */ | 476 /* Return THIS with quality dropped to AFDO. */ |
417 profile_probability afdo () const | 477 profile_probability afdo () const |
418 { | 478 { |
419 profile_probability ret = *this; | 479 profile_probability ret = *this; |
420 ret.m_quality = profile_afdo; | 480 ret.m_quality = profile_afdo; |
421 return ret; | |
422 } | |
423 | |
424 profile_probability combine_with_freq (int freq1, profile_probability other, | |
425 int freq2) const | |
426 { | |
427 profile_probability ret; | |
428 | |
429 if (*this == profile_probability::uninitialized () | |
430 || other == profile_probability::uninitialized ()) | |
431 return profile_probability::uninitialized (); | |
432 | |
433 gcc_checking_assert (freq1 >= 0 && freq2 >= 0); | |
434 if (!freq1 && !freq2) | |
435 { | |
436 ret.m_val = (m_val + other.m_val) / 2; | |
437 } | |
438 else | |
439 ret.m_val = RDIV (m_val * (uint64_t) freq1 | |
440 + other.m_val * (uint64_t) freq2, freq1 + freq2); | |
441 ret.m_quality = MIN (m_quality, other.m_quality); | |
442 return ret; | 481 return ret; |
443 } | 482 } |
444 | 483 |
445 /* Return *THIS * NUM / DEN. */ | 484 /* Return *THIS * NUM / DEN. */ |
446 profile_probability apply_scale (int64_t num, int64_t den) const | 485 profile_probability apply_scale (int64_t num, int64_t den) const |
485 } | 524 } |
486 | 525 |
487 /* Return false if profile_probability is bogus. */ | 526 /* Return false if profile_probability is bogus. */ |
488 bool verify () const | 527 bool verify () const |
489 { | 528 { |
529 gcc_checking_assert (m_quality != profile_uninitialized); | |
490 if (m_val == uninitialized_probability) | 530 if (m_val == uninitialized_probability) |
491 return m_quality == profile_guessed; | 531 return m_quality == profile_guessed; |
492 else | 532 else if (m_quality < profile_guessed) |
493 return m_val <= max_probability; | 533 return false; |
534 return m_val <= max_probability; | |
494 } | 535 } |
495 | 536 |
496 /* Comparsions are three-state and conservative. False is returned if | 537 /* Comparsions are three-state and conservative. False is returned if |
497 the inequality can not be decided. */ | 538 the inequality can not be decided. */ |
498 bool operator< (const profile_probability &other) const | 539 bool operator< (const profile_probability &other) const |
521 | 562 |
522 /* Return true if THIS is known to differ significantly from OTHER. */ | 563 /* Return true if THIS is known to differ significantly from OTHER. */ |
523 bool differs_from_p (profile_probability other) const; | 564 bool differs_from_p (profile_probability other) const; |
524 /* Return if difference is greater than 50%. */ | 565 /* Return if difference is greater than 50%. */ |
525 bool differs_lot_from_p (profile_probability other) const; | 566 bool differs_lot_from_p (profile_probability other) const; |
567 /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER | |
568 happens with COUNT2 probablity. Return probablity that either *THIS or | |
569 OTHER happens. */ | |
570 profile_probability combine_with_count (profile_count count1, | |
571 profile_probability other, | |
572 profile_count count2) const; | |
526 | 573 |
527 /* LTO streaming support. */ | 574 /* LTO streaming support. */ |
528 static profile_probability stream_in (struct lto_input_block *); | 575 static profile_probability stream_in (struct lto_input_block *); |
529 void stream_out (struct output_block *); | 576 void stream_out (struct output_block *); |
530 void stream_out (struct lto_output_stream *); | 577 void stream_out (struct lto_output_stream *); |
531 }; | 578 }; |
532 | 579 |
533 /* Main data type to hold profile counters in GCC. In most cases profile | 580 /* Main data type to hold profile counters in GCC. Profile counts originate |
534 counts originate from profile feedback. They are 64bit integers | 581 either from profile feedback, static profile estimation or both. We do not |
535 representing number of executions during the train run. | 582 perform whole program profile propagation and thus profile estimation |
583 counters are often local to function, while counters from profile feedback | |
584 (or special cases of profile estimation) can be used inter-procedurally. | |
585 | |
586 There are 3 basic types | |
587 1) local counters which are result of intra-procedural static profile | |
588 estimation. | |
589 2) ipa counters which are result of profile feedback or special case | |
590 of static profile estimation (such as in function main). | |
591 3) counters which counts as 0 inter-procedurally (beause given function | |
592 was never run in train feedback) but they hold local static profile | |
593 estimate. | |
594 | |
595 Counters of type 1 and 3 can not be mixed with counters of different type | |
596 within operation (because whole function should use one type of counter) | |
597 with exception that global zero mix in most operations where outcome is | |
598 well defined. | |
599 | |
600 To take local counter and use it inter-procedurally use ipa member function | |
601 which strips information irelevant at the inter-procedural level. | |
602 | |
603 Counters are 61bit integers representing number of executions during the | |
604 train run or normalized frequency within the function. | |
605 | |
536 As the profile is maintained during the compilation, many adjustments are | 606 As the profile is maintained during the compilation, many adjustments are |
537 made. Not all transformations can be made precisely, most importantly | 607 made. Not all transformations can be made precisely, most importantly |
538 when code is being duplicated. It also may happen that part of CFG has | 608 when code is being duplicated. It also may happen that part of CFG has |
539 profile counts known while other do not - for example when LTO optimizing | 609 profile counts known while other do not - for example when LTO optimizing |
540 partly profiled program or when profile was lost due to COMDAT merging. | 610 partly profiled program or when profile was lost due to COMDAT merging. |
559 main () function | 629 main () function |
560 profile_count::uninitialized () for unknown execution count. | 630 profile_count::uninitialized () for unknown execution count. |
561 | 631 |
562 */ | 632 */ |
563 | 633 |
634 class sreal; | |
635 | |
564 class GTY(()) profile_count | 636 class GTY(()) profile_count |
565 { | 637 { |
638 public: | |
566 /* Use 62bit to hold basic block counters. Should be at least | 639 /* Use 62bit to hold basic block counters. Should be at least |
567 64bit. Although a counter cannot be negative, we use a signed | 640 64bit. Although a counter cannot be negative, we use a signed |
568 type to hold various extra stages. */ | 641 type to hold various extra stages. */ |
569 | 642 |
570 static const int n_bits = 62; | 643 static const int n_bits = 61; |
644 private: | |
571 static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2; | 645 static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2; |
572 static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1; | 646 static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1; |
573 | 647 |
574 uint64_t m_val : n_bits; | 648 uint64_t m_val : n_bits; |
575 enum profile_quality m_quality : 2; | 649 enum profile_quality m_quality : 3; |
650 | |
651 /* Return true if both values can meaningfully appear in single function | |
652 body. We have either all counters in function local or global, otherwise | |
653 operations between them are not really defined well. */ | |
654 bool compatible_p (const profile_count other) const | |
655 { | |
656 if (!initialized_p () || !other.initialized_p ()) | |
657 return true; | |
658 if (*this == profile_count::zero () | |
659 || other == profile_count::zero ()) | |
660 return true; | |
661 return ipa_p () == other.ipa_p (); | |
662 } | |
576 public: | 663 public: |
577 | 664 |
578 /* Used for counters which are expected to be never executed. */ | 665 /* Used for counters which are expected to be never executed. */ |
579 static profile_count zero () | 666 static profile_count zero () |
580 { | 667 { |
581 return from_gcov_type (0); | 668 return from_gcov_type (0); |
669 } | |
670 static profile_count adjusted_zero () | |
671 { | |
672 profile_count c; | |
673 c.m_val = 0; | |
674 c.m_quality = profile_adjusted; | |
675 return c; | |
582 } | 676 } |
583 static profile_count guessed_zero () | 677 static profile_count guessed_zero () |
584 { | 678 { |
585 profile_count c; | 679 profile_count c; |
586 c.m_val = 0; | 680 c.m_val = 0; |
595 initialization did not happen yet or because profile is unknown. */ | 689 initialization did not happen yet or because profile is unknown. */ |
596 static profile_count uninitialized () | 690 static profile_count uninitialized () |
597 { | 691 { |
598 profile_count c; | 692 profile_count c; |
599 c.m_val = uninitialized_count; | 693 c.m_val = uninitialized_count; |
600 c.m_quality = profile_guessed; | 694 c.m_quality = profile_guessed_local; |
601 return c; | 695 return c; |
602 } | |
603 | |
604 /* The profiling runtime uses gcov_type, which is usually 64bit integer. | |
605 Conversions back and forth are used to read the coverage and get it | |
606 into internal representation. */ | |
607 static profile_count from_gcov_type (gcov_type v) | |
608 { | |
609 profile_count ret; | |
610 gcc_checking_assert (v >= 0 && (uint64_t) v <= max_count); | |
611 ret.m_val = v; | |
612 ret.m_quality = profile_precise; | |
613 return ret; | |
614 } | 696 } |
615 | 697 |
616 /* Conversion to gcov_type is lossy. */ | 698 /* Conversion to gcov_type is lossy. */ |
617 gcov_type to_gcov_type () const | 699 gcov_type to_gcov_type () const |
618 { | 700 { |
628 /* Return true if value can be trusted. */ | 710 /* Return true if value can be trusted. */ |
629 bool reliable_p () const | 711 bool reliable_p () const |
630 { | 712 { |
631 return m_quality >= profile_adjusted; | 713 return m_quality >= profile_adjusted; |
632 } | 714 } |
715 /* Return true if vlaue can be operated inter-procedurally. */ | |
716 bool ipa_p () const | |
717 { | |
718 return !initialized_p () || m_quality >= profile_guessed_global0; | |
719 } | |
720 /* Return true if quality of profile is precise. */ | |
721 bool precise_p () const | |
722 { | |
723 return m_quality == profile_precise; | |
724 } | |
725 | |
726 /* Get the quality of the count. */ | |
727 enum profile_quality quality () const { return m_quality; } | |
633 | 728 |
634 /* When merging basic blocks, the two different profile counts are unified. | 729 /* When merging basic blocks, the two different profile counts are unified. |
635 Return true if this can be done without losing info about profile. | 730 Return true if this can be done without losing info about profile. |
636 The only case we care about here is when first BB contains something | 731 The only case we care about here is when first BB contains something |
637 that makes it terminate in a way not visible in CFG. */ | 732 that makes it terminate in a way not visible in CFG. */ |
669 return other; | 764 return other; |
670 if (!initialized_p () || !other.initialized_p ()) | 765 if (!initialized_p () || !other.initialized_p ()) |
671 return profile_count::uninitialized (); | 766 return profile_count::uninitialized (); |
672 | 767 |
673 profile_count ret; | 768 profile_count ret; |
769 gcc_checking_assert (compatible_p (other)); | |
674 ret.m_val = m_val + other.m_val; | 770 ret.m_val = m_val + other.m_val; |
675 ret.m_quality = MIN (m_quality, other.m_quality); | 771 ret.m_quality = MIN (m_quality, other.m_quality); |
676 return ret; | 772 return ret; |
677 } | 773 } |
678 profile_count &operator+= (const profile_count &other) | 774 profile_count &operator+= (const profile_count &other) |
686 } | 782 } |
687 if (!initialized_p () || !other.initialized_p ()) | 783 if (!initialized_p () || !other.initialized_p ()) |
688 return *this = profile_count::uninitialized (); | 784 return *this = profile_count::uninitialized (); |
689 else | 785 else |
690 { | 786 { |
787 gcc_checking_assert (compatible_p (other)); | |
691 m_val += other.m_val; | 788 m_val += other.m_val; |
692 m_quality = MIN (m_quality, other.m_quality); | 789 m_quality = MIN (m_quality, other.m_quality); |
693 } | 790 } |
694 return *this; | 791 return *this; |
695 } | 792 } |
697 { | 794 { |
698 if (*this == profile_count::zero () || other == profile_count::zero ()) | 795 if (*this == profile_count::zero () || other == profile_count::zero ()) |
699 return *this; | 796 return *this; |
700 if (!initialized_p () || !other.initialized_p ()) | 797 if (!initialized_p () || !other.initialized_p ()) |
701 return profile_count::uninitialized (); | 798 return profile_count::uninitialized (); |
799 gcc_checking_assert (compatible_p (other)); | |
702 profile_count ret; | 800 profile_count ret; |
703 ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0; | 801 ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0; |
704 ret.m_quality = MIN (m_quality, other.m_quality); | 802 ret.m_quality = MIN (m_quality, other.m_quality); |
705 return ret; | 803 return ret; |
706 } | 804 } |
710 return *this; | 808 return *this; |
711 if (!initialized_p () || !other.initialized_p ()) | 809 if (!initialized_p () || !other.initialized_p ()) |
712 return *this = profile_count::uninitialized (); | 810 return *this = profile_count::uninitialized (); |
713 else | 811 else |
714 { | 812 { |
813 gcc_checking_assert (compatible_p (other)); | |
715 m_val = m_val >= other.m_val ? m_val - other.m_val: 0; | 814 m_val = m_val >= other.m_val ? m_val - other.m_val: 0; |
716 m_quality = MIN (m_quality, other.m_quality); | 815 m_quality = MIN (m_quality, other.m_quality); |
717 } | 816 } |
718 return *this; | 817 return *this; |
719 } | 818 } |
720 | 819 |
721 /* Return false if profile_count is bogus. */ | 820 /* Return false if profile_count is bogus. */ |
722 bool verify () const | 821 bool verify () const |
723 { | 822 { |
724 return m_val != uninitialized_count || m_quality == profile_guessed; | 823 gcc_checking_assert (m_quality != profile_uninitialized); |
824 return m_val != uninitialized_count || m_quality == profile_guessed_local; | |
725 } | 825 } |
726 | 826 |
727 /* Comparsions are three-state and conservative. False is returned if | 827 /* Comparsions are three-state and conservative. False is returned if |
728 the inequality can not be decided. */ | 828 the inequality can not be decided. */ |
729 bool operator< (const profile_count &other) const | 829 bool operator< (const profile_count &other) const |
730 { | 830 { |
731 return initialized_p () && other.initialized_p () && m_val < other.m_val; | 831 if (!initialized_p () || !other.initialized_p ()) |
832 return false; | |
833 if (*this == profile_count::zero ()) | |
834 return !(other == profile_count::zero ()); | |
835 if (other == profile_count::zero ()) | |
836 return false; | |
837 gcc_checking_assert (compatible_p (other)); | |
838 return m_val < other.m_val; | |
732 } | 839 } |
733 bool operator> (const profile_count &other) const | 840 bool operator> (const profile_count &other) const |
734 { | 841 { |
842 if (!initialized_p () || !other.initialized_p ()) | |
843 return false; | |
844 if (*this == profile_count::zero ()) | |
845 return false; | |
846 if (other == profile_count::zero ()) | |
847 return !(*this == profile_count::zero ()); | |
848 gcc_checking_assert (compatible_p (other)); | |
735 return initialized_p () && other.initialized_p () && m_val > other.m_val; | 849 return initialized_p () && other.initialized_p () && m_val > other.m_val; |
736 } | 850 } |
737 bool operator< (const gcov_type other) const | 851 bool operator< (const gcov_type other) const |
738 { | 852 { |
853 gcc_checking_assert (ipa_p ()); | |
739 gcc_checking_assert (other >= 0); | 854 gcc_checking_assert (other >= 0); |
740 return initialized_p () && m_val < (uint64_t) other; | 855 return initialized_p () && m_val < (uint64_t) other; |
741 } | 856 } |
742 bool operator> (const gcov_type other) const | 857 bool operator> (const gcov_type other) const |
743 { | 858 { |
859 gcc_checking_assert (ipa_p ()); | |
744 gcc_checking_assert (other >= 0); | 860 gcc_checking_assert (other >= 0); |
745 return initialized_p () && m_val > (uint64_t) other; | 861 return initialized_p () && m_val > (uint64_t) other; |
746 } | 862 } |
747 | 863 |
748 bool operator<= (const profile_count &other) const | 864 bool operator<= (const profile_count &other) const |
749 { | 865 { |
750 return initialized_p () && other.initialized_p () && m_val <= other.m_val; | 866 if (!initialized_p () || !other.initialized_p ()) |
867 return false; | |
868 if (*this == profile_count::zero ()) | |
869 return true; | |
870 if (other == profile_count::zero ()) | |
871 return (*this == profile_count::zero ()); | |
872 gcc_checking_assert (compatible_p (other)); | |
873 return m_val <= other.m_val; | |
751 } | 874 } |
752 bool operator>= (const profile_count &other) const | 875 bool operator>= (const profile_count &other) const |
753 { | 876 { |
754 return initialized_p () && other.initialized_p () && m_val >= other.m_val; | 877 if (!initialized_p () || !other.initialized_p ()) |
878 return false; | |
879 if (other == profile_count::zero ()) | |
880 return true; | |
881 if (*this == profile_count::zero ()) | |
882 return !(other == profile_count::zero ()); | |
883 gcc_checking_assert (compatible_p (other)); | |
884 return m_val >= other.m_val; | |
755 } | 885 } |
756 bool operator<= (const gcov_type other) const | 886 bool operator<= (const gcov_type other) const |
757 { | 887 { |
888 gcc_checking_assert (ipa_p ()); | |
758 gcc_checking_assert (other >= 0); | 889 gcc_checking_assert (other >= 0); |
759 return initialized_p () && m_val <= (uint64_t) other; | 890 return initialized_p () && m_val <= (uint64_t) other; |
760 } | 891 } |
761 bool operator>= (const gcov_type other) const | 892 bool operator>= (const gcov_type other) const |
762 { | 893 { |
894 gcc_checking_assert (ipa_p ()); | |
763 gcc_checking_assert (other >= 0); | 895 gcc_checking_assert (other >= 0); |
764 return initialized_p () && m_val >= (uint64_t) other; | 896 return initialized_p () && m_val >= (uint64_t) other; |
897 } | |
898 /* Return true when value is not zero and can be used for scaling. | |
899 This is different from *this > 0 because that requires counter to | |
900 be IPA. */ | |
901 bool nonzero_p () const | |
902 { | |
903 return initialized_p () && m_val != 0; | |
904 } | |
905 | |
906 /* Make counter forcingly nonzero. */ | |
907 profile_count force_nonzero () const | |
908 { | |
909 if (!initialized_p ()) | |
910 return *this; | |
911 profile_count ret = *this; | |
912 if (ret.m_val == 0) | |
913 { | |
914 ret.m_val = 1; | |
915 ret.m_quality = MIN (m_quality, profile_adjusted); | |
916 } | |
917 return ret; | |
918 } | |
919 | |
920 profile_count max (profile_count other) const | |
921 { | |
922 if (!initialized_p ()) | |
923 return other; | |
924 if (!other.initialized_p ()) | |
925 return *this; | |
926 if (*this == profile_count::zero ()) | |
927 return other; | |
928 if (other == profile_count::zero ()) | |
929 return *this; | |
930 gcc_checking_assert (compatible_p (other)); | |
931 if (m_val < other.m_val || (m_val == other.m_val | |
932 && m_quality < other.m_quality)) | |
933 return other; | |
934 return *this; | |
765 } | 935 } |
766 | 936 |
767 /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter | 937 /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter |
768 accordingly. */ | 938 accordingly. */ |
769 profile_count apply_probability (int prob) const | 939 profile_count apply_probability (int prob) const |
812 ret.m_quality = MIN (m_quality, profile_adjusted); | 982 ret.m_quality = MIN (m_quality, profile_adjusted); |
813 return ret; | 983 return ret; |
814 } | 984 } |
815 profile_count apply_scale (profile_count num, profile_count den) const | 985 profile_count apply_scale (profile_count num, profile_count den) const |
816 { | 986 { |
817 if (m_val == 0) | 987 if (*this == profile_count::zero ()) |
818 return *this; | 988 return *this; |
819 if (num.m_val == 0) | 989 if (num == profile_count::zero ()) |
820 return num; | 990 return num; |
821 if (!initialized_p () || !num.initialized_p () || !den.initialized_p ()) | 991 if (!initialized_p () || !num.initialized_p () || !den.initialized_p ()) |
822 return profile_count::uninitialized (); | 992 return profile_count::uninitialized (); |
823 gcc_checking_assert (den > 0); | |
824 if (num == den) | 993 if (num == den) |
825 return *this; | 994 return *this; |
995 gcc_checking_assert (den.m_val); | |
826 | 996 |
827 profile_count ret; | 997 profile_count ret; |
828 uint64_t val; | 998 uint64_t val; |
829 safe_scale_64bit (m_val, num.m_val, den.m_val, &val); | 999 safe_scale_64bit (m_val, num.m_val, den.m_val, &val); |
830 ret.m_val = MIN (val, max_count); | 1000 ret.m_val = MIN (val, max_count); |
831 ret.m_quality = MIN (m_quality, profile_adjusted); | 1001 ret.m_quality = MIN (MIN (MIN (m_quality, profile_adjusted), |
1002 num.m_quality), den.m_quality); | |
1003 if (num.ipa_p () && !ret.ipa_p ()) | |
1004 ret.m_quality = MIN (num.m_quality, profile_guessed); | |
1005 return ret; | |
1006 } | |
1007 | |
1008 /* Return THIS with quality dropped to GUESSED_LOCAL. */ | |
1009 profile_count guessed_local () const | |
1010 { | |
1011 profile_count ret = *this; | |
1012 if (!initialized_p ()) | |
1013 return *this; | |
1014 ret.m_quality = profile_guessed_local; | |
1015 return ret; | |
1016 } | |
1017 | |
1018 /* We know that profile is globally 0 but keep local profile if present. */ | |
1019 profile_count global0 () const | |
1020 { | |
1021 profile_count ret = *this; | |
1022 if (!initialized_p ()) | |
1023 return *this; | |
1024 ret.m_quality = profile_guessed_global0; | |
1025 return ret; | |
1026 } | |
1027 | |
1028 /* We know that profile is globally adjusted 0 but keep local profile | |
1029 if present. */ | |
1030 profile_count global0adjusted () const | |
1031 { | |
1032 profile_count ret = *this; | |
1033 if (!initialized_p ()) | |
1034 return *this; | |
1035 ret.m_quality = profile_guessed_global0adjusted; | |
832 return ret; | 1036 return ret; |
833 } | 1037 } |
834 | 1038 |
835 /* Return THIS with quality dropped to GUESSED. */ | 1039 /* Return THIS with quality dropped to GUESSED. */ |
836 profile_count guessed () const | 1040 profile_count guessed () const |
837 { | 1041 { |
838 profile_count ret = *this; | 1042 profile_count ret = *this; |
839 ret.m_quality = profile_guessed; | 1043 ret.m_quality = MIN (ret.m_quality, profile_guessed); |
840 return ret; | 1044 return ret; |
1045 } | |
1046 | |
1047 /* Return variant of profile counte which is always safe to compare | |
1048 acorss functions. */ | |
1049 profile_count ipa () const | |
1050 { | |
1051 if (m_quality > profile_guessed_global0adjusted) | |
1052 return *this; | |
1053 if (m_quality == profile_guessed_global0) | |
1054 return profile_count::zero (); | |
1055 if (m_quality == profile_guessed_global0adjusted) | |
1056 return profile_count::adjusted_zero (); | |
1057 return profile_count::uninitialized (); | |
841 } | 1058 } |
842 | 1059 |
843 /* Return THIS with quality dropped to AFDO. */ | 1060 /* Return THIS with quality dropped to AFDO. */ |
844 profile_count afdo () const | 1061 profile_count afdo () const |
845 { | 1062 { |
850 | 1067 |
851 /* Return probability of event with counter THIS within event with counter | 1068 /* Return probability of event with counter THIS within event with counter |
852 OVERALL. */ | 1069 OVERALL. */ |
853 profile_probability probability_in (const profile_count overall) const | 1070 profile_probability probability_in (const profile_count overall) const |
854 { | 1071 { |
855 if (!m_val) | 1072 if (*this == profile_count::zero () |
1073 && !(overall == profile_count::zero ())) | |
856 return profile_probability::never (); | 1074 return profile_probability::never (); |
857 if (!initialized_p () || !overall.initialized_p () | 1075 if (!initialized_p () || !overall.initialized_p () |
858 || !overall.m_val) | 1076 || !overall.m_val) |
859 return profile_probability::uninitialized (); | 1077 return profile_probability::uninitialized (); |
860 profile_probability ret; | 1078 if (*this == overall && m_quality == profile_precise) |
861 if (overall < m_val) | 1079 return profile_probability::always (); |
862 ret.m_val = profile_probability::max_probability; | 1080 profile_probability ret; |
1081 gcc_checking_assert (compatible_p (overall)); | |
1082 | |
1083 if (overall.m_val < m_val) | |
1084 { | |
1085 ret.m_val = profile_probability::max_probability; | |
1086 ret.m_quality = profile_guessed; | |
1087 return ret; | |
1088 } | |
863 else | 1089 else |
864 ret.m_val = RDIV (m_val * profile_probability::max_probability, | 1090 ret.m_val = RDIV (m_val * profile_probability::max_probability, |
865 overall.m_val); | 1091 overall.m_val); |
866 ret.m_quality = MIN (m_quality, overall.m_quality); | 1092 ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality), |
867 return ret; | 1093 profile_guessed), profile_adjusted); |
868 } | 1094 return ret; |
1095 } | |
1096 | |
1097 int to_frequency (struct function *fun) const; | |
1098 int to_cgraph_frequency (profile_count entry_bb_count) const; | |
1099 sreal to_sreal_scale (profile_count in, bool *known = NULL) const; | |
869 | 1100 |
870 /* Output THIS to F. */ | 1101 /* Output THIS to F. */ |
871 void dump (FILE *f) const; | 1102 void dump (FILE *f) const; |
872 | 1103 |
873 /* Print THIS to stderr. */ | 1104 /* Print THIS to stderr. */ |
874 void debug () const; | 1105 void debug () const; |
875 | 1106 |
876 /* Return true if THIS is known to differ significantly from OTHER. */ | 1107 /* Return true if THIS is known to differ significantly from OTHER. */ |
877 bool differs_from_p (profile_count other) const; | 1108 bool differs_from_p (profile_count other) const; |
1109 | |
1110 /* We want to scale profile across function boundary from NUM to DEN. | |
1111 Take care of the side case when NUM and DEN are zeros of incompatible | |
1112 kinds. */ | |
1113 static void adjust_for_ipa_scaling (profile_count *num, profile_count *den); | |
1114 | |
1115 /* THIS is a count of bb which is known to be executed IPA times. | |
1116 Combine this information into bb counter. This means returning IPA | |
1117 if it is nonzero, not changing anything if IPA is uninitialized | |
1118 and if IPA is zero, turning THIS into corresponding local profile with | |
1119 global0. */ | |
1120 profile_count combine_with_ipa_count (profile_count ipa); | |
1121 | |
1122 /* The profiling runtime uses gcov_type, which is usually 64bit integer. | |
1123 Conversions back and forth are used to read the coverage and get it | |
1124 into internal representation. */ | |
1125 static profile_count from_gcov_type (gcov_type v); | |
878 | 1126 |
879 /* LTO streaming support. */ | 1127 /* LTO streaming support. */ |
880 static profile_count stream_in (struct lto_input_block *); | 1128 static profile_count stream_in (struct lto_input_block *); |
881 void stream_out (struct output_block *); | 1129 void stream_out (struct output_block *); |
882 void stream_out (struct lto_output_stream *); | 1130 void stream_out (struct lto_output_stream *); |