comparison gcc/profile-count.h @ 132:d34655255c78

update gcc-8.2
author mir3636
date Thu, 25 Oct 2018 10:21:07 +0900
parents 84e7813d76e9
children 1830386684a0
comparison
equal deleted inserted replaced
130:e108057fa461 132:d34655255c78
1 /* Profile counter container type. 1 /* Profile counter container type.
2 Copyright (C) 2017 Free Software Foundation, Inc. 2 Copyright (C) 2017-2018 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka 3 Contributed by Jan Hubicka
4 4
5 This file is part of GCC. 5 This file is part of GCC.
6 6
7 GCC is free software; you can redistribute it and/or modify it under 7 GCC is free software; you can redistribute it and/or modify it under
19 <http://www.gnu.org/licenses/>. */ 19 <http://www.gnu.org/licenses/>. */
20 20
21 #ifndef GCC_PROFILE_COUNT_H 21 #ifndef GCC_PROFILE_COUNT_H
22 #define GCC_PROFILE_COUNT_H 22 #define GCC_PROFILE_COUNT_H
23 23
24 struct function;
25 class profile_count;
26
24 /* Quality of the profile count. Because gengtype does not support enums 27 /* Quality of the profile count. Because gengtype does not support enums
25 inside of classes, this is in global namespace. */ 28 inside of classes, this is in global namespace. */
26 enum profile_quality { 29 enum profile_quality {
30 /* Uninitialized value. */
31 profile_uninitialized,
32 /* Profile is based on static branch prediction heuristics and may
33 or may not match reality. It is local to function and can not be compared
34 inter-procedurally. Never used by probabilities (they are always local).
35 */
36 profile_guessed_local,
37 /* Profile was read by feedback and was 0, we used local heuristics to guess
38 better. This is the case of functions not run in profile fedback.
39 Never used by probabilities. */
40 profile_guessed_global0,
41
42 /* Same as profile_guessed_global0 but global count is adjusted 0. */
43 profile_guessed_global0adjusted,
44
27 /* Profile is based on static branch prediction heuristics. It may or may 45 /* Profile is based on static branch prediction heuristics. It may or may
28 not reflect the reality. */ 46 not reflect the reality but it can be compared interprocedurally
29 profile_guessed = 0, 47 (for example, we inlined function w/o profile feedback into function
48 with feedback and propagated from that).
49 Never used by probablities. */
50 profile_guessed,
30 /* Profile was determined by autofdo. */ 51 /* Profile was determined by autofdo. */
31 profile_afdo = 1, 52 profile_afdo,
32 /* Profile was originally based on feedback but it was adjusted 53 /* Profile was originally based on feedback but it was adjusted
33 by code duplicating optimization. It may not precisely reflect the 54 by code duplicating optimization. It may not precisely reflect the
34 particular code path. */ 55 particular code path. */
35 profile_adjusted = 2, 56 profile_adjusted,
36 /* Profile was read from profile feedback or determined by accurate static 57 /* Profile was read from profile feedback or determined by accurate static
37 method. */ 58 method. */
38 profile_precise = 3 59 profile_precise
39 }; 60 };
61
62 extern const char *profile_quality_as_string (enum profile_quality);
40 63
41 /* The base value for branch probability notes and edge probabilities. */ 64 /* The base value for branch probability notes and edge probabilities. */
42 #define REG_BR_PROB_BASE 10000 65 #define REG_BR_PROB_BASE 10000
43 66
44 #define RDIV(X,Y) (((X) + (Y) / 2) / (Y)) 67 #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
112 these conversions will probably go away because they are lossy. 135 these conversions will probably go away because they are lossy.
113 */ 136 */
114 137
115 class GTY((user)) profile_probability 138 class GTY((user)) profile_probability
116 { 139 {
117 static const int n_bits = 30; 140 static const int n_bits = 29;
118 /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that 141 /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
119 will lead to harder multiplication sequences. */ 142 will lead to harder multiplication sequences. */
120 static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2); 143 static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
121 static const uint32_t uninitialized_probability 144 static const uint32_t uninitialized_probability
122 = ((uint32_t) 1 << (n_bits - 1)) - 1; 145 = ((uint32_t) 1 << (n_bits - 1)) - 1;
123 146
124 uint32_t m_val : 30; 147 uint32_t m_val : 29;
125 enum profile_quality m_quality : 2; 148 enum profile_quality m_quality : 3;
126 149
127 friend class profile_count; 150 friend class profile_count;
128 public: 151 public:
129 152
130 /* Named probabilities. */ 153 /* Named probabilities. */
144 } 167 }
145 static profile_probability very_unlikely () 168 static profile_probability very_unlikely ()
146 { 169 {
147 /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */ 170 /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
148 profile_probability r 171 profile_probability r
149 = profile_probability::always ().apply_scale (1, 2000); 172 = profile_probability::guessed_always ().apply_scale (1, 2000);
150 r.m_val--; 173 r.m_val--;
151 return r; 174 return r;
152 } 175 }
153 static profile_probability unlikely () 176 static profile_probability unlikely ()
154 { 177 {
155 /* Be consistent with PROB_VERY_LIKELY in predict.h. */ 178 /* Be consistent with PROB_VERY_LIKELY in predict.h. */
156 profile_probability r 179 profile_probability r
157 = profile_probability::always ().apply_scale (1, 5); 180 = profile_probability::guessed_always ().apply_scale (1, 5);
158 r.m_val--; 181 r.m_val--;
159 return r; 182 return r;
160 } 183 }
161 static profile_probability even () 184 static profile_probability even ()
162 { 185 {
163 return profile_probability::always ().apply_scale (1, 2); 186 return profile_probability::guessed_always ().apply_scale (1, 2);
164 } 187 }
165 static profile_probability very_likely () 188 static profile_probability very_likely ()
166 { 189 {
167 return profile_probability::always () - very_unlikely (); 190 return profile_probability::always () - very_unlikely ();
168 } 191 }
224 247
225 /* Conversion to and from RTL representation of profile probabilities. */ 248 /* Conversion to and from RTL representation of profile probabilities. */
226 static profile_probability from_reg_br_prob_note (int v) 249 static profile_probability from_reg_br_prob_note (int v)
227 { 250 {
228 profile_probability ret; 251 profile_probability ret;
229 ret.m_val = ((unsigned int)v) / 4; 252 ret.m_val = ((unsigned int)v) / 8;
230 ret.m_quality = (enum profile_quality)(v & 3); 253 ret.m_quality = (enum profile_quality)(v & 7);
231 return ret; 254 return ret;
232 } 255 }
233 int to_reg_br_prob_note () const 256 int to_reg_br_prob_note () const
234 { 257 {
235 gcc_checking_assert (initialized_p ()); 258 gcc_checking_assert (initialized_p ());
236 int ret = m_val * 4 + m_quality; 259 int ret = m_val * 8 + m_quality;
237 gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret) 260 gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret)
238 == *this); 261 == *this);
239 return ret; 262 return ret;
240 } 263 }
241 264
328 return profile_probability::never (); 351 return profile_probability::never ();
329 if (!initialized_p () || !other.initialized_p ()) 352 if (!initialized_p () || !other.initialized_p ())
330 return profile_probability::uninitialized (); 353 return profile_probability::uninitialized ();
331 profile_probability ret; 354 profile_probability ret;
332 ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); 355 ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
333 ret.m_quality = MIN (m_quality, other.m_quality); 356 ret.m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted);
334 return ret; 357 return ret;
335 } 358 }
336 profile_probability &operator*= (const profile_probability &other) 359 profile_probability &operator*= (const profile_probability &other)
337 { 360 {
338 if (*this == profile_probability::never () 361 if (*this == profile_probability::never ()
341 if (!initialized_p () || !other.initialized_p ()) 364 if (!initialized_p () || !other.initialized_p ())
342 return *this = profile_probability::uninitialized (); 365 return *this = profile_probability::uninitialized ();
343 else 366 else
344 { 367 {
345 m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability); 368 m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
346 m_quality = MIN (m_quality, other.m_quality); 369 m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted);
347 } 370 }
348 return *this; 371 return *this;
349 } 372 }
350 profile_probability operator/ (const profile_probability &other) const 373 profile_probability operator/ (const profile_probability &other) const
351 { 374 {
352 if (*this == profile_probability::never ()) 375 if (*this == profile_probability::never ())
353 return profile_probability::never (); 376 return profile_probability::never ();
354 if (!initialized_p () || !other.initialized_p ()) 377 if (!initialized_p () || !other.initialized_p ())
355 return profile_probability::uninitialized (); 378 return profile_probability::uninitialized ();
356 profile_probability ret; 379 profile_probability ret;
380 /* If we get probability above 1, mark it as unreliable and return 1. */
357 if (m_val >= other.m_val) 381 if (m_val >= other.m_val)
358 ret.m_val = max_probability; 382 {
383 ret.m_val = max_probability;
384 ret.m_quality = MIN (MIN (m_quality, other.m_quality),
385 profile_guessed);
386 return ret;
387 }
359 else if (!m_val) 388 else if (!m_val)
360 ret.m_val = 0; 389 ret.m_val = 0;
361 else 390 else
362 { 391 {
363 gcc_checking_assert (other.m_val); 392 gcc_checking_assert (other.m_val);
364 ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability, 393 ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
365 other.m_val), 394 other.m_val),
366 max_probability); 395 max_probability);
367 } 396 }
368 ret.m_quality = MIN (m_quality, other.m_quality); 397 ret.m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted);
369 return ret; 398 return ret;
370 } 399 }
371 profile_probability &operator/= (const profile_probability &other) 400 profile_probability &operator/= (const profile_probability &other)
372 { 401 {
373 if (*this == profile_probability::never ()) 402 if (*this == profile_probability::never ())
374 return *this = profile_probability::never (); 403 return *this = profile_probability::never ();
375 if (!initialized_p () || !other.initialized_p ()) 404 if (!initialized_p () || !other.initialized_p ())
376 return *this = profile_probability::uninitialized (); 405 return *this = profile_probability::uninitialized ();
377 else 406 else
378 { 407 {
408 /* If we get probability above 1, mark it as unreliable
409 and return 1. */
379 if (m_val > other.m_val) 410 if (m_val > other.m_val)
380 m_val = max_probability; 411 {
412 m_val = max_probability;
413 m_quality = MIN (MIN (m_quality, other.m_quality),
414 profile_guessed);
415 return *this;
416 }
381 else if (!m_val) 417 else if (!m_val)
382 ; 418 ;
383 else 419 else
384 { 420 {
385 gcc_checking_assert (other.m_val); 421 gcc_checking_assert (other.m_val);
386 m_val = MIN (RDIV ((uint64_t)m_val * max_probability, 422 m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
387 other.m_val), 423 other.m_val),
388 max_probability); 424 max_probability);
389 } 425 }
390 m_quality = MIN (m_quality, other.m_quality); 426 m_quality = MIN (MIN (m_quality, other.m_quality), profile_adjusted);
391 } 427 }
392 return *this; 428 return *this;
429 }
430
431 /* Split *THIS (ORIG) probability into 2 probabilities, such that
432 the returned one (FIRST) is *THIS * CPROB and *THIS is
433 adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
434 == ORIG. This is useful e.g. when splitting a conditional
435 branch like:
436 if (cond)
437 goto lab; // ORIG probability
438 into
439 if (cond1)
440 goto lab; // FIRST = ORIG * CPROB probability
441 if (cond2)
442 goto lab; // SECOND probability
443 such that the overall probability of jumping to lab remains
444 the same. CPROB gives the relative probability between the
445 branches. */
446 profile_probability split (const profile_probability &cprob)
447 {
448 profile_probability ret = *this * cprob;
449 /* The following is equivalent to:
450 *this = cprob.invert () * *this / ret.invert (); */
451 *this = (*this - ret) / ret.invert ();
452 return ret;
393 } 453 }
394 454
395 gcov_type apply (gcov_type val) const 455 gcov_type apply (gcov_type val) const
396 { 456 {
397 if (*this == profile_probability::uninitialized ()) 457 if (*this == profile_probability::uninitialized ())
416 /* Return THIS with quality dropped to AFDO. */ 476 /* Return THIS with quality dropped to AFDO. */
417 profile_probability afdo () const 477 profile_probability afdo () const
418 { 478 {
419 profile_probability ret = *this; 479 profile_probability ret = *this;
420 ret.m_quality = profile_afdo; 480 ret.m_quality = profile_afdo;
421 return ret;
422 }
423
424 profile_probability combine_with_freq (int freq1, profile_probability other,
425 int freq2) const
426 {
427 profile_probability ret;
428
429 if (*this == profile_probability::uninitialized ()
430 || other == profile_probability::uninitialized ())
431 return profile_probability::uninitialized ();
432
433 gcc_checking_assert (freq1 >= 0 && freq2 >= 0);
434 if (!freq1 && !freq2)
435 {
436 ret.m_val = (m_val + other.m_val) / 2;
437 }
438 else
439 ret.m_val = RDIV (m_val * (uint64_t) freq1
440 + other.m_val * (uint64_t) freq2, freq1 + freq2);
441 ret.m_quality = MIN (m_quality, other.m_quality);
442 return ret; 481 return ret;
443 } 482 }
444 483
445 /* Return *THIS * NUM / DEN. */ 484 /* Return *THIS * NUM / DEN. */
446 profile_probability apply_scale (int64_t num, int64_t den) const 485 profile_probability apply_scale (int64_t num, int64_t den) const
485 } 524 }
486 525
487 /* Return false if profile_probability is bogus. */ 526 /* Return false if profile_probability is bogus. */
488 bool verify () const 527 bool verify () const
489 { 528 {
529 gcc_checking_assert (m_quality != profile_uninitialized);
490 if (m_val == uninitialized_probability) 530 if (m_val == uninitialized_probability)
491 return m_quality == profile_guessed; 531 return m_quality == profile_guessed;
492 else 532 else if (m_quality < profile_guessed)
493 return m_val <= max_probability; 533 return false;
534 return m_val <= max_probability;
494 } 535 }
495 536
496 /* Comparsions are three-state and conservative. False is returned if 537 /* Comparsions are three-state and conservative. False is returned if
497 the inequality can not be decided. */ 538 the inequality can not be decided. */
498 bool operator< (const profile_probability &other) const 539 bool operator< (const profile_probability &other) const
521 562
522 /* Return true if THIS is known to differ significantly from OTHER. */ 563 /* Return true if THIS is known to differ significantly from OTHER. */
523 bool differs_from_p (profile_probability other) const; 564 bool differs_from_p (profile_probability other) const;
524 /* Return if difference is greater than 50%. */ 565 /* Return if difference is greater than 50%. */
525 bool differs_lot_from_p (profile_probability other) const; 566 bool differs_lot_from_p (profile_probability other) const;
567 /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
568 happens with COUNT2 probablity. Return probablity that either *THIS or
569 OTHER happens. */
570 profile_probability combine_with_count (profile_count count1,
571 profile_probability other,
572 profile_count count2) const;
526 573
527 /* LTO streaming support. */ 574 /* LTO streaming support. */
528 static profile_probability stream_in (struct lto_input_block *); 575 static profile_probability stream_in (struct lto_input_block *);
529 void stream_out (struct output_block *); 576 void stream_out (struct output_block *);
530 void stream_out (struct lto_output_stream *); 577 void stream_out (struct lto_output_stream *);
531 }; 578 };
532 579
533 /* Main data type to hold profile counters in GCC. In most cases profile 580 /* Main data type to hold profile counters in GCC. Profile counts originate
534 counts originate from profile feedback. They are 64bit integers 581 either from profile feedback, static profile estimation or both. We do not
535 representing number of executions during the train run. 582 perform whole program profile propagation and thus profile estimation
583 counters are often local to function, while counters from profile feedback
584 (or special cases of profile estimation) can be used inter-procedurally.
585
586 There are 3 basic types
587 1) local counters which are result of intra-procedural static profile
588 estimation.
589 2) ipa counters which are result of profile feedback or special case
590 of static profile estimation (such as in function main).
591 3) counters which counts as 0 inter-procedurally (beause given function
592 was never run in train feedback) but they hold local static profile
593 estimate.
594
595 Counters of type 1 and 3 can not be mixed with counters of different type
596 within operation (because whole function should use one type of counter)
597 with exception that global zero mix in most operations where outcome is
598 well defined.
599
600 To take local counter and use it inter-procedurally use ipa member function
601 which strips information irelevant at the inter-procedural level.
602
603 Counters are 61bit integers representing number of executions during the
604 train run or normalized frequency within the function.
605
536 As the profile is maintained during the compilation, many adjustments are 606 As the profile is maintained during the compilation, many adjustments are
537 made. Not all transformations can be made precisely, most importantly 607 made. Not all transformations can be made precisely, most importantly
538 when code is being duplicated. It also may happen that part of CFG has 608 when code is being duplicated. It also may happen that part of CFG has
539 profile counts known while other do not - for example when LTO optimizing 609 profile counts known while other do not - for example when LTO optimizing
540 partly profiled program or when profile was lost due to COMDAT merging. 610 partly profiled program or when profile was lost due to COMDAT merging.
559 main () function 629 main () function
560 profile_count::uninitialized () for unknown execution count. 630 profile_count::uninitialized () for unknown execution count.
561 631
562 */ 632 */
563 633
634 class sreal;
635
564 class GTY(()) profile_count 636 class GTY(()) profile_count
565 { 637 {
638 public:
566 /* Use 62bit to hold basic block counters. Should be at least 639 /* Use 62bit to hold basic block counters. Should be at least
567 64bit. Although a counter cannot be negative, we use a signed 640 64bit. Although a counter cannot be negative, we use a signed
568 type to hold various extra stages. */ 641 type to hold various extra stages. */
569 642
570 static const int n_bits = 62; 643 static const int n_bits = 61;
644 private:
571 static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2; 645 static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
572 static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1; 646 static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
573 647
574 uint64_t m_val : n_bits; 648 uint64_t m_val : n_bits;
575 enum profile_quality m_quality : 2; 649 enum profile_quality m_quality : 3;
650
651 /* Return true if both values can meaningfully appear in single function
652 body. We have either all counters in function local or global, otherwise
653 operations between them are not really defined well. */
654 bool compatible_p (const profile_count other) const
655 {
656 if (!initialized_p () || !other.initialized_p ())
657 return true;
658 if (*this == profile_count::zero ()
659 || other == profile_count::zero ())
660 return true;
661 return ipa_p () == other.ipa_p ();
662 }
576 public: 663 public:
577 664
578 /* Used for counters which are expected to be never executed. */ 665 /* Used for counters which are expected to be never executed. */
579 static profile_count zero () 666 static profile_count zero ()
580 { 667 {
581 return from_gcov_type (0); 668 return from_gcov_type (0);
669 }
670 static profile_count adjusted_zero ()
671 {
672 profile_count c;
673 c.m_val = 0;
674 c.m_quality = profile_adjusted;
675 return c;
582 } 676 }
583 static profile_count guessed_zero () 677 static profile_count guessed_zero ()
584 { 678 {
585 profile_count c; 679 profile_count c;
586 c.m_val = 0; 680 c.m_val = 0;
595 initialization did not happen yet or because profile is unknown. */ 689 initialization did not happen yet or because profile is unknown. */
596 static profile_count uninitialized () 690 static profile_count uninitialized ()
597 { 691 {
598 profile_count c; 692 profile_count c;
599 c.m_val = uninitialized_count; 693 c.m_val = uninitialized_count;
600 c.m_quality = profile_guessed; 694 c.m_quality = profile_guessed_local;
601 return c; 695 return c;
602 }
603
604 /* The profiling runtime uses gcov_type, which is usually 64bit integer.
605 Conversions back and forth are used to read the coverage and get it
606 into internal representation. */
607 static profile_count from_gcov_type (gcov_type v)
608 {
609 profile_count ret;
610 gcc_checking_assert (v >= 0 && (uint64_t) v <= max_count);
611 ret.m_val = v;
612 ret.m_quality = profile_precise;
613 return ret;
614 } 696 }
615 697
616 /* Conversion to gcov_type is lossy. */ 698 /* Conversion to gcov_type is lossy. */
617 gcov_type to_gcov_type () const 699 gcov_type to_gcov_type () const
618 { 700 {
628 /* Return true if value can be trusted. */ 710 /* Return true if value can be trusted. */
629 bool reliable_p () const 711 bool reliable_p () const
630 { 712 {
631 return m_quality >= profile_adjusted; 713 return m_quality >= profile_adjusted;
632 } 714 }
715 /* Return true if vlaue can be operated inter-procedurally. */
716 bool ipa_p () const
717 {
718 return !initialized_p () || m_quality >= profile_guessed_global0;
719 }
720 /* Return true if quality of profile is precise. */
721 bool precise_p () const
722 {
723 return m_quality == profile_precise;
724 }
725
726 /* Get the quality of the count. */
727 enum profile_quality quality () const { return m_quality; }
633 728
634 /* When merging basic blocks, the two different profile counts are unified. 729 /* When merging basic blocks, the two different profile counts are unified.
635 Return true if this can be done without losing info about profile. 730 Return true if this can be done without losing info about profile.
636 The only case we care about here is when first BB contains something 731 The only case we care about here is when first BB contains something
637 that makes it terminate in a way not visible in CFG. */ 732 that makes it terminate in a way not visible in CFG. */
669 return other; 764 return other;
670 if (!initialized_p () || !other.initialized_p ()) 765 if (!initialized_p () || !other.initialized_p ())
671 return profile_count::uninitialized (); 766 return profile_count::uninitialized ();
672 767
673 profile_count ret; 768 profile_count ret;
769 gcc_checking_assert (compatible_p (other));
674 ret.m_val = m_val + other.m_val; 770 ret.m_val = m_val + other.m_val;
675 ret.m_quality = MIN (m_quality, other.m_quality); 771 ret.m_quality = MIN (m_quality, other.m_quality);
676 return ret; 772 return ret;
677 } 773 }
678 profile_count &operator+= (const profile_count &other) 774 profile_count &operator+= (const profile_count &other)
686 } 782 }
687 if (!initialized_p () || !other.initialized_p ()) 783 if (!initialized_p () || !other.initialized_p ())
688 return *this = profile_count::uninitialized (); 784 return *this = profile_count::uninitialized ();
689 else 785 else
690 { 786 {
787 gcc_checking_assert (compatible_p (other));
691 m_val += other.m_val; 788 m_val += other.m_val;
692 m_quality = MIN (m_quality, other.m_quality); 789 m_quality = MIN (m_quality, other.m_quality);
693 } 790 }
694 return *this; 791 return *this;
695 } 792 }
697 { 794 {
698 if (*this == profile_count::zero () || other == profile_count::zero ()) 795 if (*this == profile_count::zero () || other == profile_count::zero ())
699 return *this; 796 return *this;
700 if (!initialized_p () || !other.initialized_p ()) 797 if (!initialized_p () || !other.initialized_p ())
701 return profile_count::uninitialized (); 798 return profile_count::uninitialized ();
799 gcc_checking_assert (compatible_p (other));
702 profile_count ret; 800 profile_count ret;
703 ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0; 801 ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
704 ret.m_quality = MIN (m_quality, other.m_quality); 802 ret.m_quality = MIN (m_quality, other.m_quality);
705 return ret; 803 return ret;
706 } 804 }
710 return *this; 808 return *this;
711 if (!initialized_p () || !other.initialized_p ()) 809 if (!initialized_p () || !other.initialized_p ())
712 return *this = profile_count::uninitialized (); 810 return *this = profile_count::uninitialized ();
713 else 811 else
714 { 812 {
813 gcc_checking_assert (compatible_p (other));
715 m_val = m_val >= other.m_val ? m_val - other.m_val: 0; 814 m_val = m_val >= other.m_val ? m_val - other.m_val: 0;
716 m_quality = MIN (m_quality, other.m_quality); 815 m_quality = MIN (m_quality, other.m_quality);
717 } 816 }
718 return *this; 817 return *this;
719 } 818 }
720 819
721 /* Return false if profile_count is bogus. */ 820 /* Return false if profile_count is bogus. */
722 bool verify () const 821 bool verify () const
723 { 822 {
724 return m_val != uninitialized_count || m_quality == profile_guessed; 823 gcc_checking_assert (m_quality != profile_uninitialized);
824 return m_val != uninitialized_count || m_quality == profile_guessed_local;
725 } 825 }
726 826
727 /* Comparsions are three-state and conservative. False is returned if 827 /* Comparsions are three-state and conservative. False is returned if
728 the inequality can not be decided. */ 828 the inequality can not be decided. */
729 bool operator< (const profile_count &other) const 829 bool operator< (const profile_count &other) const
730 { 830 {
731 return initialized_p () && other.initialized_p () && m_val < other.m_val; 831 if (!initialized_p () || !other.initialized_p ())
832 return false;
833 if (*this == profile_count::zero ())
834 return !(other == profile_count::zero ());
835 if (other == profile_count::zero ())
836 return false;
837 gcc_checking_assert (compatible_p (other));
838 return m_val < other.m_val;
732 } 839 }
733 bool operator> (const profile_count &other) const 840 bool operator> (const profile_count &other) const
734 { 841 {
842 if (!initialized_p () || !other.initialized_p ())
843 return false;
844 if (*this == profile_count::zero ())
845 return false;
846 if (other == profile_count::zero ())
847 return !(*this == profile_count::zero ());
848 gcc_checking_assert (compatible_p (other));
735 return initialized_p () && other.initialized_p () && m_val > other.m_val; 849 return initialized_p () && other.initialized_p () && m_val > other.m_val;
736 } 850 }
737 bool operator< (const gcov_type other) const 851 bool operator< (const gcov_type other) const
738 { 852 {
853 gcc_checking_assert (ipa_p ());
739 gcc_checking_assert (other >= 0); 854 gcc_checking_assert (other >= 0);
740 return initialized_p () && m_val < (uint64_t) other; 855 return initialized_p () && m_val < (uint64_t) other;
741 } 856 }
742 bool operator> (const gcov_type other) const 857 bool operator> (const gcov_type other) const
743 { 858 {
859 gcc_checking_assert (ipa_p ());
744 gcc_checking_assert (other >= 0); 860 gcc_checking_assert (other >= 0);
745 return initialized_p () && m_val > (uint64_t) other; 861 return initialized_p () && m_val > (uint64_t) other;
746 } 862 }
747 863
748 bool operator<= (const profile_count &other) const 864 bool operator<= (const profile_count &other) const
749 { 865 {
750 return initialized_p () && other.initialized_p () && m_val <= other.m_val; 866 if (!initialized_p () || !other.initialized_p ())
867 return false;
868 if (*this == profile_count::zero ())
869 return true;
870 if (other == profile_count::zero ())
871 return (*this == profile_count::zero ());
872 gcc_checking_assert (compatible_p (other));
873 return m_val <= other.m_val;
751 } 874 }
752 bool operator>= (const profile_count &other) const 875 bool operator>= (const profile_count &other) const
753 { 876 {
754 return initialized_p () && other.initialized_p () && m_val >= other.m_val; 877 if (!initialized_p () || !other.initialized_p ())
878 return false;
879 if (other == profile_count::zero ())
880 return true;
881 if (*this == profile_count::zero ())
882 return !(other == profile_count::zero ());
883 gcc_checking_assert (compatible_p (other));
884 return m_val >= other.m_val;
755 } 885 }
756 bool operator<= (const gcov_type other) const 886 bool operator<= (const gcov_type other) const
757 { 887 {
888 gcc_checking_assert (ipa_p ());
758 gcc_checking_assert (other >= 0); 889 gcc_checking_assert (other >= 0);
759 return initialized_p () && m_val <= (uint64_t) other; 890 return initialized_p () && m_val <= (uint64_t) other;
760 } 891 }
761 bool operator>= (const gcov_type other) const 892 bool operator>= (const gcov_type other) const
762 { 893 {
894 gcc_checking_assert (ipa_p ());
763 gcc_checking_assert (other >= 0); 895 gcc_checking_assert (other >= 0);
764 return initialized_p () && m_val >= (uint64_t) other; 896 return initialized_p () && m_val >= (uint64_t) other;
897 }
898 /* Return true when value is not zero and can be used for scaling.
899 This is different from *this > 0 because that requires counter to
900 be IPA. */
901 bool nonzero_p () const
902 {
903 return initialized_p () && m_val != 0;
904 }
905
906 /* Make counter forcingly nonzero. */
907 profile_count force_nonzero () const
908 {
909 if (!initialized_p ())
910 return *this;
911 profile_count ret = *this;
912 if (ret.m_val == 0)
913 {
914 ret.m_val = 1;
915 ret.m_quality = MIN (m_quality, profile_adjusted);
916 }
917 return ret;
918 }
919
920 profile_count max (profile_count other) const
921 {
922 if (!initialized_p ())
923 return other;
924 if (!other.initialized_p ())
925 return *this;
926 if (*this == profile_count::zero ())
927 return other;
928 if (other == profile_count::zero ())
929 return *this;
930 gcc_checking_assert (compatible_p (other));
931 if (m_val < other.m_val || (m_val == other.m_val
932 && m_quality < other.m_quality))
933 return other;
934 return *this;
765 } 935 }
766 936
767 /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter 937 /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
768 accordingly. */ 938 accordingly. */
769 profile_count apply_probability (int prob) const 939 profile_count apply_probability (int prob) const
812 ret.m_quality = MIN (m_quality, profile_adjusted); 982 ret.m_quality = MIN (m_quality, profile_adjusted);
813 return ret; 983 return ret;
814 } 984 }
815 profile_count apply_scale (profile_count num, profile_count den) const 985 profile_count apply_scale (profile_count num, profile_count den) const
816 { 986 {
817 if (m_val == 0) 987 if (*this == profile_count::zero ())
818 return *this; 988 return *this;
819 if (num.m_val == 0) 989 if (num == profile_count::zero ())
820 return num; 990 return num;
821 if (!initialized_p () || !num.initialized_p () || !den.initialized_p ()) 991 if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
822 return profile_count::uninitialized (); 992 return profile_count::uninitialized ();
823 gcc_checking_assert (den > 0);
824 if (num == den) 993 if (num == den)
825 return *this; 994 return *this;
995 gcc_checking_assert (den.m_val);
826 996
827 profile_count ret; 997 profile_count ret;
828 uint64_t val; 998 uint64_t val;
829 safe_scale_64bit (m_val, num.m_val, den.m_val, &val); 999 safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
830 ret.m_val = MIN (val, max_count); 1000 ret.m_val = MIN (val, max_count);
831 ret.m_quality = MIN (m_quality, profile_adjusted); 1001 ret.m_quality = MIN (MIN (MIN (m_quality, profile_adjusted),
1002 num.m_quality), den.m_quality);
1003 if (num.ipa_p () && !ret.ipa_p ())
1004 ret.m_quality = MIN (num.m_quality, profile_guessed);
1005 return ret;
1006 }
1007
1008 /* Return THIS with quality dropped to GUESSED_LOCAL. */
1009 profile_count guessed_local () const
1010 {
1011 profile_count ret = *this;
1012 if (!initialized_p ())
1013 return *this;
1014 ret.m_quality = profile_guessed_local;
1015 return ret;
1016 }
1017
1018 /* We know that profile is globally 0 but keep local profile if present. */
1019 profile_count global0 () const
1020 {
1021 profile_count ret = *this;
1022 if (!initialized_p ())
1023 return *this;
1024 ret.m_quality = profile_guessed_global0;
1025 return ret;
1026 }
1027
1028 /* We know that profile is globally adjusted 0 but keep local profile
1029 if present. */
1030 profile_count global0adjusted () const
1031 {
1032 profile_count ret = *this;
1033 if (!initialized_p ())
1034 return *this;
1035 ret.m_quality = profile_guessed_global0adjusted;
832 return ret; 1036 return ret;
833 } 1037 }
834 1038
835 /* Return THIS with quality dropped to GUESSED. */ 1039 /* Return THIS with quality dropped to GUESSED. */
836 profile_count guessed () const 1040 profile_count guessed () const
837 { 1041 {
838 profile_count ret = *this; 1042 profile_count ret = *this;
839 ret.m_quality = profile_guessed; 1043 ret.m_quality = MIN (ret.m_quality, profile_guessed);
840 return ret; 1044 return ret;
1045 }
1046
1047 /* Return variant of profile counte which is always safe to compare
1048 acorss functions. */
1049 profile_count ipa () const
1050 {
1051 if (m_quality > profile_guessed_global0adjusted)
1052 return *this;
1053 if (m_quality == profile_guessed_global0)
1054 return profile_count::zero ();
1055 if (m_quality == profile_guessed_global0adjusted)
1056 return profile_count::adjusted_zero ();
1057 return profile_count::uninitialized ();
841 } 1058 }
842 1059
843 /* Return THIS with quality dropped to AFDO. */ 1060 /* Return THIS with quality dropped to AFDO. */
844 profile_count afdo () const 1061 profile_count afdo () const
845 { 1062 {
850 1067
851 /* Return probability of event with counter THIS within event with counter 1068 /* Return probability of event with counter THIS within event with counter
852 OVERALL. */ 1069 OVERALL. */
853 profile_probability probability_in (const profile_count overall) const 1070 profile_probability probability_in (const profile_count overall) const
854 { 1071 {
855 if (!m_val) 1072 if (*this == profile_count::zero ()
1073 && !(overall == profile_count::zero ()))
856 return profile_probability::never (); 1074 return profile_probability::never ();
857 if (!initialized_p () || !overall.initialized_p () 1075 if (!initialized_p () || !overall.initialized_p ()
858 || !overall.m_val) 1076 || !overall.m_val)
859 return profile_probability::uninitialized (); 1077 return profile_probability::uninitialized ();
860 profile_probability ret; 1078 if (*this == overall && m_quality == profile_precise)
861 if (overall < m_val) 1079 return profile_probability::always ();
862 ret.m_val = profile_probability::max_probability; 1080 profile_probability ret;
1081 gcc_checking_assert (compatible_p (overall));
1082
1083 if (overall.m_val < m_val)
1084 {
1085 ret.m_val = profile_probability::max_probability;
1086 ret.m_quality = profile_guessed;
1087 return ret;
1088 }
863 else 1089 else
864 ret.m_val = RDIV (m_val * profile_probability::max_probability, 1090 ret.m_val = RDIV (m_val * profile_probability::max_probability,
865 overall.m_val); 1091 overall.m_val);
866 ret.m_quality = MIN (m_quality, overall.m_quality); 1092 ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
867 return ret; 1093 profile_guessed), profile_adjusted);
868 } 1094 return ret;
1095 }
1096
1097 int to_frequency (struct function *fun) const;
1098 int to_cgraph_frequency (profile_count entry_bb_count) const;
1099 sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
869 1100
870 /* Output THIS to F. */ 1101 /* Output THIS to F. */
871 void dump (FILE *f) const; 1102 void dump (FILE *f) const;
872 1103
873 /* Print THIS to stderr. */ 1104 /* Print THIS to stderr. */
874 void debug () const; 1105 void debug () const;
875 1106
876 /* Return true if THIS is known to differ significantly from OTHER. */ 1107 /* Return true if THIS is known to differ significantly from OTHER. */
877 bool differs_from_p (profile_count other) const; 1108 bool differs_from_p (profile_count other) const;
1109
1110 /* We want to scale profile across function boundary from NUM to DEN.
1111 Take care of the side case when NUM and DEN are zeros of incompatible
1112 kinds. */
1113 static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1114
1115 /* THIS is a count of bb which is known to be executed IPA times.
1116 Combine this information into bb counter. This means returning IPA
1117 if it is nonzero, not changing anything if IPA is uninitialized
1118 and if IPA is zero, turning THIS into corresponding local profile with
1119 global0. */
1120 profile_count combine_with_ipa_count (profile_count ipa);
1121
1122 /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1123 Conversions back and forth are used to read the coverage and get it
1124 into internal representation. */
1125 static profile_count from_gcov_type (gcov_type v);
878 1126
879 /* LTO streaming support. */ 1127 /* LTO streaming support. */
880 static profile_count stream_in (struct lto_input_block *); 1128 static profile_count stream_in (struct lto_input_block *);
881 void stream_out (struct output_block *); 1129 void stream_out (struct output_block *);
882 void stream_out (struct lto_output_stream *); 1130 void stream_out (struct lto_output_stream *);