--- a/gcc/ipa-inline.c	Fri Feb 12 23:41:23 2010 +0900
+++ b/gcc/ipa-inline.c	Mon May 24 12:47:05 2010 +0900
@@ -1,5 +1,6 @@
 /* Inlining decision heuristics.
-   Copyright (C) 2003, 2004, 2007, 2008, 2009 Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010
+   Free Software Foundation, Inc.
    Contributed by Jan Hubicka
 
 This file is part of GCC.
@@ -127,6 +128,7 @@
 #include "flags.h"
 #include "cgraph.h"
 #include "diagnostic.h"
+#include "gimple-pretty-print.h"
 #include "timevar.h"
 #include "params.h"
 #include "fibheap.h"
@@ -159,9 +161,10 @@
   INLINE_SIZE,
   INLINE_ALL
 };
+
 static bool
-cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode,
-				      int);
+cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode);
+static void cgraph_flatten (struct cgraph_node *node);
 
 
 /* Statistics we collect about inlining algorithm.  */
@@ -266,7 +269,8 @@
       else
 	{
 	  struct cgraph_node *n;
-	  n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+	  n = cgraph_clone_node (e->callee, e->callee->decl,
+				 e->count, e->frequency, e->loop_nest,
 				 update_original, NULL);
 	  cgraph_redirect_edge_callee (e, n);
 	}
@@ -283,6 +287,7 @@
       + inline_summary (e->callee)->estimated_self_stack_size;
   if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
     e->callee->global.inlined_to->global.estimated_stack_size = peak;
+  cgraph_propagate_frequency (e->callee);
 
   /* Recursively clone all bodies.  */
   for (e = e->callee->callees; e; e = e->next_callee)
@@ -304,20 +309,11 @@
   struct cgraph_node *to = NULL, *what;
   struct cgraph_edge *curr = e;
   int freq;
-  bool duplicate = false;
-  int orig_size = e->callee->global.size;
 
   gcc_assert (e->inline_failed);
   e->inline_failed = CIF_OK;
-
-  if (!e->callee->global.inlined)
-    DECL_POSSIBLY_INLINED (e->callee->decl) = true;
-  e->callee->global.inlined = true;
+  DECL_POSSIBLY_INLINED (e->callee->decl) = true;
 
-  if (e->callee->callers->next_caller
-      || !cgraph_can_remove_if_no_direct_calls_p (e->callee)
-      || e->callee->same_comdat_group)
-    duplicate = true;
   cgraph_clone_inlined_nodes (e, true, update_original);
 
   what = e->callee;
@@ -335,8 +331,6 @@
   gcc_assert (what->global.inlined_to == to);
   if (new_size > old_size)
     overall_size += new_size - old_size;
-  if (!duplicate)
-    overall_size -= orig_size;
   ncalls_inlined++;
 
   if (flag_indirect_inlining)
@@ -345,11 +339,9 @@
     return false;
 }
 
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
-   Return following unredirected edge in the list of callers
-   of EDGE->CALLEE  */
+/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.  */
 
-static struct cgraph_edge *
+static void
 cgraph_mark_inline (struct cgraph_edge *edge)
 {
   struct cgraph_node *to = edge->caller;
@@ -369,8 +361,6 @@
 	    edge = next;
 	}
     }
-
-  return edge;
 }
 
 /* Estimate the growth caused by inlining NODE into all callees.  */
@@ -478,6 +468,9 @@
 {
   tree decl = n->decl;
 
+  if (n->local.disregard_inline_limits)
+    return true;
+
   if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl))
     {
       if (reason)
@@ -543,23 +536,57 @@
    of the function or function body size.  */
 
 static int
-cgraph_edge_badness (struct cgraph_edge *edge)
+cgraph_edge_badness (struct cgraph_edge *edge, bool dump)
 {
   gcov_type badness;
   int growth =
-    cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+    (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+     - edge->caller->global.size);
+
+  if (edge->callee->local.disregard_inline_limits)
+    return INT_MIN;
 
-  growth -= edge->caller->global.size;
+  if (dump)
+    {
+      fprintf (dump_file, "    Badness calculcation for %s -> %s\n",
+	       cgraph_node_name (edge->caller),
+	       cgraph_node_name (edge->callee));
+      fprintf (dump_file, "      growth %i, time %i-%i, size %i-%i\n",
+	       growth,
+	       edge->callee->global.time,
+	       inline_summary (edge->callee)->time_inlining_benefit,
+	       edge->callee->global.size,
+	       inline_summary (edge->callee)->size_inlining_benefit);
+    }
 
   /* Always prefer inlining saving code size.  */
   if (growth <= 0)
-    badness = INT_MIN - growth;
+    {
+      badness = INT_MIN - growth;
+      if (dump)
+	fprintf (dump_file, "      %i: Growth %i < 0\n", (int) badness,
+		 growth);
+    }
 
   /* When profiling is available, base priorities -(#calls / growth).
      So we optimize for overall number of "executed" inlined calls.  */
   else if (max_count)
-    badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1))
-    	      * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+    {
+      badness =
+	((int)
+	 ((double) edge->count * INT_MIN / max_count / (max_benefit + 1)) *
+	 (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+      if (dump)
+	{
+	  fprintf (dump_file,
+		   "      %i (relative %f): profile info. Relative count %f"
+		   " * Relative benefit %f\n",
+		   (int) badness, (double) badness / INT_MIN,
+		   (double) edge->count / max_count,
+		   (double) (inline_summary (edge->callee)->
+			     time_inlining_benefit + 1) / (max_benefit + 1));
+	}
+    }
 
   /* When function local profile is available, base priorities on
      growth / frequency, so we optimize for overall frequency of inlined
@@ -573,9 +600,13 @@
   else if (flag_guess_branch_prob)
     {
       int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1;
+      int benefitperc;
+      int growth_for_all;
       badness = growth * 10000;
-      div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
-      	          / (edge->callee->global.time + 1) + 1, 100);
+      benefitperc =
+	MIN (100 * inline_summary (edge->callee)->time_inlining_benefit /
+	     (edge->callee->global.time + 1) +1, 100);
+      div *= benefitperc;
 
 
       /* Decrease badness if call is nested.  */
@@ -586,9 +617,17 @@
 	div = 1;
       if (badness > 0)
 	badness /= div;
-      badness += cgraph_estimate_growth (edge->callee);
+      growth_for_all = cgraph_estimate_growth (edge->callee);
+      badness += growth_for_all;
       if (badness > INT_MAX)
-        badness = INT_MAX;
+	badness = INT_MAX;
+      if (dump)
+	{
+	  fprintf (dump_file,
+		   "      %i: guessed profile. frequency %i, overall growth %i,"
+		   " benefit %i%%, divisor %i\n",
+		   (int) badness, edge->frequency, growth_for_all, benefitperc, div);
+	}
     }
   /* When function local profile is not available or it does not give
      useful information (ie frequency is zero), base the cost on
@@ -603,10 +642,17 @@
       if (badness > 0)
 	badness >>= nest;
       else
-        {
+	{
 	  badness <<= nest;
-        }
+	}
+      if (dump)
+	fprintf (dump_file, "      %i: no profile. nest %i\n", (int) badness,
+		 nest);
     }
+
+  /* Ensure that we did not overflow in all the fixed point math above.  */
+  gcc_assert (badness >= INT_MIN);
+  gcc_assert (badness <= INT_MAX - 1);
   /* Make recursive inlining happen always after other inlining is done.  */
   if (cgraph_recursive_inlining_p (edge->caller, edge->callee, NULL))
     return badness + 1;
@@ -623,7 +669,7 @@
   struct cgraph_edge *edge;
   cgraph_inline_failed_t failed_reason;
 
-  if (!node->local.inlinable || node->local.disregard_inline_limits
+  if (!node->local.inlinable
       || node->global.inlined_to)
     return;
   if (bitmap_bit_p (updated_nodes, node->uid))
@@ -650,7 +696,7 @@
   for (edge = node->callers; edge; edge = edge->next_caller)
     if (edge->inline_failed)
       {
-	int badness = cgraph_edge_badness (edge);
+	int badness = cgraph_edge_badness (edge, false);
 	if (edge->aux)
 	  {
 	    fibnode_t n = (fibnode_t) edge->aux;
@@ -659,8 +705,12 @@
 	      continue;
 
 	    /* fibheap_replace_key only increase the keys.  */
-	    if (fibheap_replace_key (heap, n, badness))
-	      continue;
+	    if (badness < n->key)
+	      {
+		fibheap_replace_key (heap, n, badness);
+		gcc_assert (n->key == badness);
+	        continue;
+	      }
 	    fibheap_delete_node (heap, (fibnode_t) edge->aux);
 	  }
 	edge->aux = fibheap_insert (heap, badness, edge);
@@ -726,6 +776,12 @@
   int depth = 0;
   int n = 0;
 
+  /* It does not make sense to recursively inline always-inline functions
+     as we are going to sorry() on the remaining calls anyway.  */
+  if (node->local.disregard_inline_limits
+      && lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)))
+    return false;
+
   if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))
       || (!flag_inline_functions && !DECL_DECLARED_INLINE_P (node->decl)))
     return false;
@@ -754,7 +810,8 @@
 	     cgraph_node_name (node));
 
   /* We need original clone to copy around.  */
-  master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1,
+  master_clone = cgraph_clone_node (node, node->decl,
+				    node->count, CGRAPH_FREQ_BASE, 1,
   				    false, NULL);
   master_clone->needed = true;
   for (e = master_clone->callees; e; e = e->next_callee)
@@ -882,7 +939,7 @@
       struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
 
       gcc_assert (!edge->aux);
-      edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+      edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
     }
 }
 
@@ -915,8 +972,7 @@
 
   for (node = cgraph_nodes; node; node = node->next)
     {
-      if (!node->local.inlinable || !node->callers
-	  || node->local.disregard_inline_limits)
+      if (!node->local.inlinable || !node->callers)
 	continue;
       if (dump_file)
 	fprintf (dump_file, "Considering inline candidate %s.\n", cgraph_node_name (node));
@@ -932,7 +988,7 @@
 	if (edge->inline_failed)
 	  {
 	    gcc_assert (!edge->aux);
-	    edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+	    edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
 	  }
     }
 
@@ -940,15 +996,26 @@
   min_size = overall_size;
 
   while (overall_size <= max_size
-	 && (edge = (struct cgraph_edge *) fibheap_extract_min (heap)))
+	 && !fibheap_empty (heap))
     {
       int old_size = overall_size;
-      struct cgraph_node *where;
-      int growth =
-	cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+      struct cgraph_node *where, *callee;
+      int badness = fibheap_min_key (heap);
+      int growth;
       cgraph_inline_failed_t not_good = CIF_OK;
 
-      growth -= edge->caller->global.size;
+      edge = (struct cgraph_edge *) fibheap_extract_min (heap);
+      gcc_assert (edge->aux);
+      edge->aux = NULL;
+      if (!edge->inline_failed)
+	continue;
+#ifdef ENABLE_CHECKING
+      gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+      callee = edge->callee;
+
+      growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+		- edge->caller->global.size);
 
       if (dump_file)
 	{
@@ -961,18 +1028,17 @@
 		   " Estimated growth after inlined into all callees is %+i insns.\n"
 		   " Estimated badness is %i, frequency %.2f.\n",
 		   cgraph_node_name (edge->caller),
-		   gimple_filename ((const_gimple) edge->call_stmt),
-		   gimple_lineno ((const_gimple) edge->call_stmt),
+		   flag_wpa ? "unknown"
+		   : gimple_filename ((const_gimple) edge->call_stmt),
+		   flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
 		   cgraph_estimate_growth (edge->callee),
-		   cgraph_edge_badness (edge),
+		   badness,
 		   edge->frequency / (double)CGRAPH_FREQ_BASE);
 	  if (edge->count)
 	    fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+	  if (dump_flags & TDF_DETAILS)
+	    cgraph_edge_badness (edge, true);
 	}
-      gcc_assert (edge->aux);
-      edge->aux = NULL;
-      if (!edge->inline_failed)
-	continue;
 
       /* When not having profile info ready we don't weight by any way the
          position of call in procedure itself.  This means if call of
@@ -1008,12 +1074,14 @@
 	    }
 	}
 
-      if (!cgraph_maybe_hot_edge_p (edge))
+      if (edge->callee->local.disregard_inline_limits)
+	;
+      else if (!cgraph_maybe_hot_edge_p (edge))
  	not_good = CIF_UNLIKELY_CALL;
-      if (!flag_inline_functions
+      else if (!flag_inline_functions
 	  && !DECL_DECLARED_INLINE_P (edge->callee->decl))
  	not_good = CIF_NOT_DECLARED_INLINED;
-      if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
+      else if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
  	not_good = CIF_OPTIMIZING_FOR_SIZE;
       if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0)
 	{
@@ -1090,6 +1158,11 @@
 	 called by function we inlined (since number of it inlinable callers
 	 might change).  */
       update_caller_keys (heap, where, updated_nodes);
+
+      /* We removed one call of the function we just inlined.  If offline
+	 copy is still needed, be sure to update the keys.  */
+      if (callee != where && !callee->global.inlined_to)
+        update_caller_keys (heap, callee, updated_nodes);
       bitmap_clear (updated_nodes);
 
       if (dump_file)
@@ -1111,10 +1184,40 @@
 	    fprintf (dump_file, "New minimal size reached: %i\n", min_size);
 	}
     }
-  while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL)
+  while (!fibheap_empty (heap))
     {
+      int badness = fibheap_min_key (heap);
+
+      edge = (struct cgraph_edge *) fibheap_extract_min (heap);
       gcc_assert (edge->aux);
       edge->aux = NULL;
+      if (!edge->inline_failed)
+	continue;
+#ifdef ENABLE_CHECKING
+      gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+      if (dump_file)
+	{
+	  fprintf (dump_file,
+		   "\nSkipping %s with %i size\n",
+		   cgraph_node_name (edge->callee),
+		   edge->callee->global.size);
+	  fprintf (dump_file,
+		   " called by %s in %s:%i\n"
+		   " Estimated growth after inlined into all callees is %+i insns.\n"
+		   " Estimated badness is %i, frequency %.2f.\n",
+		   cgraph_node_name (edge->caller),
+		   flag_wpa ? "unknown"
+		   : gimple_filename ((const_gimple) edge->call_stmt),
+		   flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
+		   cgraph_estimate_growth (edge->callee),
+		   badness,
+		   edge->frequency / (double)CGRAPH_FREQ_BASE);
+	  if (edge->count)
+	    fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+	  if (dump_flags & TDF_DETAILS)
+	    cgraph_edge_badness (edge, true);
+	}
       if (!edge->callee->local.disregard_inline_limits && edge->inline_failed
           && !cgraph_recursive_inlining_p (edge->caller, edge->callee,
 				           &edge->inline_failed))
@@ -1127,6 +1230,86 @@
   BITMAP_FREE (updated_nodes);
 }
 
+/* Flatten NODE from the IPA inliner.  */
+
+static void
+cgraph_flatten (struct cgraph_node *node)
+{
+  struct cgraph_edge *e;
+
+  /* We shouldn't be called recursively when we are being processed.  */
+  gcc_assert (node->aux == NULL);
+
+  node->aux = (void *)(size_t) INLINE_ALL;
+
+  for (e = node->callees; e; e = e->next_callee)
+    {
+      struct cgraph_node *orig_callee;
+
+      if (e->call_stmt_cannot_inline_p)
+	continue;
+
+      if (!e->callee->analyzed)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Not inlining: Function body not available.\n");
+	  continue;
+	}
+
+      /* We've hit cycle?  It is time to give up.  */
+      if (e->callee->aux)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Not inlining %s into %s to avoid cycle.\n",
+		     cgraph_node_name (e->callee),
+		     cgraph_node_name (e->caller));
+	  e->inline_failed = CIF_RECURSIVE_INLINING;
+	  continue;
+	}
+
+      /* When the edge is already inlined, we just need to recurse into
+	 it in order to fully flatten the leaves.  */
+      if (!e->inline_failed)
+	{
+	  cgraph_flatten (e->callee);
+	  continue;
+	}
+
+      if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Not inlining: recursive call.\n");
+	  continue;
+	}
+
+      if (!tree_can_inline_p (e))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Not inlining: %s",
+		     cgraph_inline_failed_string (e->inline_failed));
+	  continue;
+	}
+
+      /* Inline the edge and flatten the inline clone.  Avoid
+         recursing through the original node if the node was cloned.  */
+      if (dump_file)
+	fprintf (dump_file, " Inlining %s into %s.\n",
+		 cgraph_node_name (e->callee),
+		 cgraph_node_name (e->caller));
+      orig_callee = e->callee;
+      cgraph_mark_inline_edge (e, true, NULL);
+      if (e->callee != orig_callee)
+	orig_callee->aux = (void *)(size_t) INLINE_ALL;
+      cgraph_flatten (e->callee);
+      if (e->callee != orig_callee)
+	orig_callee->aux = NULL;
+    }
+
+  node->aux = NULL;
+}
+
 /* Decide on the inlining.  We do so in the topological order to avoid
    expenses on updating data structures.  */
 
@@ -1139,12 +1322,13 @@
     XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
   int old_size = 0;
   int i;
-  bool redo_always_inline = true;
   int initial_size = 0;
 
   cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
   if (in_lto_p && flag_indirect_inlining)
     ipa_update_after_lto_read ();
+  if (flag_indirect_inlining)
+    ipa_create_all_structures_for_iinln ();
 
   max_count = 0;
   max_benefit = 0;
@@ -1177,65 +1361,29 @@
     node->aux = 0;
 
   if (dump_file)
-    fprintf (dump_file, "\nInlining always_inline functions:\n");
+    fprintf (dump_file, "\nFlattening functions:\n");
 
-  /* In the first pass mark all always_inline edges.  Do this with a priority
-     so none of our later choices will make this impossible.  */
-  while (redo_always_inline)
+  /* In the first pass handle functions to be flattened.  Do this with
+     a priority so none of our later choices will make this impossible.  */
+  for (i = nnodes - 1; i >= 0; i--)
     {
-      redo_always_inline = false;
-      for (i = nnodes - 1; i >= 0; i--)
-	{
-	  struct cgraph_edge *e, *next;
-
-	  node = order[i];
+      node = order[i];
 
-	  /* Handle nodes to be flattened, but don't update overall unit
-	     size.  */
-	  if (lookup_attribute ("flatten",
-				DECL_ATTRIBUTES (node->decl)) != NULL)
-	    {
-	      if (dump_file)
-		fprintf (dump_file,
-			 "Flattening %s\n", cgraph_node_name (node));
-	      cgraph_decide_inlining_incrementally (node, INLINE_ALL, 0);
-	    }
-
-	  if (!node->local.disregard_inline_limits)
-	    continue;
+      /* Handle nodes to be flattened, but don't update overall unit
+	 size.  Calling the incremental inliner here is lame,
+	 a simple worklist should be enough.  What should be left
+	 here from the early inliner (if it runs) is cyclic cases.
+	 Ideally when processing callees we stop inlining at the
+	 entry of cycles, possibly cloning that entry point and
+	 try to flatten itself turning it into a self-recursive
+	 function.  */
+      if (lookup_attribute ("flatten",
+			    DECL_ATTRIBUTES (node->decl)) != NULL)
+	{
 	  if (dump_file)
 	    fprintf (dump_file,
-		     "\nConsidering %s size:%i (always inline)\n",
-		     cgraph_node_name (node), node->global.size);
-	  old_size = overall_size;
-	  for (e = node->callers; e; e = next)
-	    {
-	      next = e->next_caller;
-	      if (!e->inline_failed || e->call_stmt_cannot_inline_p)
-		continue;
-	      if (cgraph_recursive_inlining_p (e->caller, e->callee,
-					       &e->inline_failed))
-		continue;
-	      if (!tree_can_inline_p (e))
-                continue;
-	      if (cgraph_mark_inline_edge (e, true, NULL))
-		redo_always_inline = true;
-	      if (dump_file)
-		fprintf (dump_file,
-			 " Inlined into %s which now has size %i.\n",
-			 cgraph_node_name (e->caller),
-			 e->caller->global.size);
-	    }
-	  /* Inlining self recursive function might introduce new calls to
-	     themselves we didn't see in the loop above.  Fill in the proper
-	     reason why inline failed.  */
-	  for (e = node->callers; e; e = e->next_caller)
-	    if (e->inline_failed)
-	      e->inline_failed = CIF_RECURSIVE_INLINING;
-	  if (dump_file)
-	    fprintf (dump_file,
-		     " Inlined for a net change of %+i size.\n",
-		     overall_size - old_size);
+		     "Flattening %s\n", cgraph_node_name (node));
+	  cgraph_flatten (node);
 	}
     }
 
@@ -1278,13 +1426,14 @@
 	      if (cgraph_check_inline_limits (node->callers->caller, node,
 					      &reason, false))
 		{
+		  struct cgraph_node *caller = node->callers->caller;
 		  cgraph_mark_inline (node->callers);
 		  if (dump_file)
 		    fprintf (dump_file,
 			     " Inlined into %s which now has %i size"
 			     " for a net change of %+i size.\n",
-			     cgraph_node_name (node->callers->caller),
-			     node->callers->caller->global.size,
+			     cgraph_node_name (caller),
+			     caller->global.size,
 			     overall_size - old_size);
 		}
 	      else
@@ -1300,7 +1449,7 @@
 
   /* Free ipa-prop structures if they are no longer needed.  */
   if (flag_indirect_inlining)
-    free_all_ipa_structures_after_iinln ();
+    ipa_free_all_structures_after_iinln ();
 
   if (dump_file)
     fprintf (dump_file,
@@ -1312,86 +1461,6 @@
   return 0;
 }
 
-/* Try to inline edge E from incremental inliner.  MODE specifies mode
-   of inliner.
-
-   We are detecting cycles by storing mode of inliner into cgraph_node last
-   time we visited it in the recursion.  In general when mode is set, we have
-   recursive inlining, but as an special case, we want to try harder inline
-   ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
-   flatten, b being always inline.  Flattening 'a' will collapse
-   a->b->c before hitting cycle.  To accommodate always inline, we however
-   need to inline a->b->c->b.
-
-   So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
-   stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode.  */
-static bool
-try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth)
-{
-  struct cgraph_node *callee = e->callee;
-  enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux;
-  bool always_inline = e->callee->local.disregard_inline_limits;
-  bool inlined = false;
-
-  /* We've hit cycle?  */
-  if (callee_mode)
-    {
-      /* It is first time we see it and we are not in ALWAY_INLINE only
-	 mode yet.  and the function in question is always_inline.  */
-      if (always_inline && mode != INLINE_ALWAYS_INLINE)
-	{
-	  if (dump_file)
-	    {
-	      indent_to (dump_file, depth);
-	      fprintf (dump_file,
-		       "Hit cycle in %s, switching to always inline only.\n",
-		       cgraph_node_name (callee));
-	    }
-	  mode = INLINE_ALWAYS_INLINE;
-	}
-      /* Otherwise it is time to give up.  */
-      else
-	{
-	  if (dump_file)
-	    {
-	      indent_to (dump_file, depth);
-	      fprintf (dump_file,
-		       "Not inlining %s into %s to avoid cycle.\n",
-		       cgraph_node_name (callee),
-		       cgraph_node_name (e->caller));
-	    }
-	  e->inline_failed = (e->callee->local.disregard_inline_limits
-		              ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
-          return false;
-	}
-    }
-
-  callee->aux = (void *)(size_t) mode;
-  if (dump_file)
-    {
-      indent_to (dump_file, depth);
-      fprintf (dump_file, " Inlining %s into %s.\n",
-	       cgraph_node_name (e->callee),
-	       cgraph_node_name (e->caller));
-    }
-  if (e->inline_failed)
-    {
-      cgraph_mark_inline (e);
-
-      /* In order to fully inline always_inline functions, we need to
-	 recurse here, since the inlined functions might not be processed by
-	 incremental inlining at all yet.
-
-	 Also flattening needs to be done recursively.  */
-
-      if (mode == INLINE_ALL || always_inline)
-	cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1);
-      inlined = true;
-    }
-  callee->aux = (void *)(size_t) callee_mode;
-  return inlined;
-}
-
 /* Return true when N is leaf function.  Accept cheap (pure&const) builtins
    in leaf functions.  */
 static bool
@@ -1407,38 +1476,29 @@
 }
 
 /* Decide on the inlining.  We do so in the topological order to avoid
-   expenses on updating data structures.
-   DEPTH is depth of recursion, used only for debug output.  */
+   expenses on updating data structures.  */
 
 static bool
 cgraph_decide_inlining_incrementally (struct cgraph_node *node,
-				      enum inlining_mode mode,
-				      int depth)
+				      enum inlining_mode mode)
 {
   struct cgraph_edge *e;
   bool inlined = false;
   cgraph_inline_failed_t failed_reason;
-  enum inlining_mode old_mode;
 
 #ifdef ENABLE_CHECKING
   verify_cgraph_node (node);
 #endif
 
-  old_mode = (enum inlining_mode) (size_t)node->aux;
-
   if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE
       && lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL)
     {
       if (dump_file)
-	{
-	  indent_to (dump_file, depth);
-	  fprintf (dump_file, "Flattening %s\n", cgraph_node_name (node));
-	}
+	fprintf (dump_file, "Incrementally flattening %s\n",
+		 cgraph_node_name (node));
       mode = INLINE_ALL;
     }
 
-  node->aux = (void *)(size_t) mode;
-
   /* First of all look for always inline functions.  */
   if (mode != INLINE_SIZE_NORECURSIVE)
     for (e = node->callees; e; e = e->next_callee)
@@ -1448,157 +1508,141 @@
 	  continue;
 	if (e->call_stmt_cannot_inline_p)
 	  continue;
-	/* When the edge is already inlined, we just need to recurse into
-	   it in order to fully flatten the leaves.  */
-	if (!e->inline_failed && mode == INLINE_ALL)
-	  {
-	    inlined |= try_inline (e, mode, depth);
-	    continue;
-	  }
 	if (dump_file)
-	  {
-	    indent_to (dump_file, depth);
-	    fprintf (dump_file,
-		     "Considering to always inline inline candidate %s.\n",
-		     cgraph_node_name (e->callee));
-	  }
+	  fprintf (dump_file,
+		   "Considering to always inline inline candidate %s.\n",
+		   cgraph_node_name (e->callee));
 	if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
 	  {
 	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: recursive call.\n");
-	      }
+	      fprintf (dump_file, "Not inlining: recursive call.\n");
 	    continue;
 	  }
 	if (!tree_can_inline_p (e))
 	  {
 	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file,
-			 "Not inlining: %s",
-                         cgraph_inline_failed_string (e->inline_failed));
-	      }
+	      fprintf (dump_file,
+		       "Not inlining: %s",
+		       cgraph_inline_failed_string (e->inline_failed));
 	    continue;
 	  }
 	if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
 	    != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
 	  {
 	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: SSA form does not match.\n");
-	      }
+	      fprintf (dump_file, "Not inlining: SSA form does not match.\n");
 	    continue;
 	  }
 	if (!e->callee->analyzed)
 	  {
 	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file,
-			 "Not inlining: Function body no longer available.\n");
-	      }
+	      fprintf (dump_file,
+		       "Not inlining: Function body no longer available.\n");
 	    continue;
 	  }
-	inlined |= try_inline (e, mode, depth);
+
+	if (dump_file)
+	  fprintf (dump_file, " Inlining %s into %s.\n",
+		   cgraph_node_name (e->callee),
+		   cgraph_node_name (e->caller));
+	cgraph_mark_inline (e);
+	inlined = true;
       }
 
   /* Now do the automatic inlining.  */
-  if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE)
-    for (e = node->callees; e; e = e->next_callee)
-      {
-        int allowed_growth = 0;
-	if (!e->callee->local.inlinable
-	    || !e->inline_failed
-	    || e->callee->local.disregard_inline_limits)
-	  continue;
-	if (dump_file)
-	  fprintf (dump_file, "Considering inline candidate %s.\n",
-		   cgraph_node_name (e->callee));
-	if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: recursive call.\n");
-	      }
+  if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE
+      /* Never inline regular functions into always-inline functions
+	 during incremental inlining.  */
+      && !node->local.disregard_inline_limits)
+    {
+      bitmap visited = BITMAP_ALLOC (NULL);
+      for (e = node->callees; e; e = e->next_callee)
+	{
+	  int allowed_growth = 0;
+	  if (!e->callee->local.inlinable
+	      || !e->inline_failed
+	      || e->callee->local.disregard_inline_limits)
+	    continue;
+	  /* We are inlining a function to all call-sites in node
+	     or to none.  So visit each candidate only once.  */
+	  if (!bitmap_set_bit (visited, e->callee->uid))
 	    continue;
-	  }
-	if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
-	    != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: SSA form does not match.\n");
-	      }
-	    continue;
-	  }
+	  if (dump_file)
+	    fprintf (dump_file, "Considering inline candidate %s.\n",
+		     cgraph_node_name (e->callee));
+	  if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "Not inlining: recursive call.\n");
+	      continue;
+	    }
+	  if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+	      != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+	    {
+	      if (dump_file)
+		fprintf (dump_file,
+			 "Not inlining: SSA form does not match.\n");
+	      continue;
+	    }
 
-	if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
-	    && optimize_function_for_speed_p (cfun))
-	  allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+	  if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
+	      && optimize_function_for_speed_p (cfun))
+	    allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
 
-	/* When the function body would grow and inlining the function won't
-	   eliminate the need for offline copy of the function, don't inline.
-	 */
-	if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
-	     || (!flag_inline_functions
-		 && !DECL_DECLARED_INLINE_P (e->callee->decl)))
-	    && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
-		> e->caller->global.size + allowed_growth)
-	    && cgraph_estimate_growth (e->callee) > allowed_growth)
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
+	  /* When the function body would grow and inlining the function
+	     won't eliminate the need for offline copy of the function,
+	     don't inline.  */
+	  if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
+	       || (!flag_inline_functions
+		   && !DECL_DECLARED_INLINE_P (e->callee->decl)))
+	      && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+		  > e->caller->global.size + allowed_growth)
+	      && cgraph_estimate_growth (e->callee) > allowed_growth)
+	    {
+	      if (dump_file)
 		fprintf (dump_file,
 			 "Not inlining: code size would grow by %i.\n",
 			 cgraph_estimate_size_after_inlining (1, e->caller,
 							      e->callee)
 			 - e->caller->global.size);
-	      }
-	    continue;
-	  }
-	if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
-				         false)
-	    || e->call_stmt_cannot_inline_p)
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
+	      continue;
+	    }
+	  if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+					   false)
+	      || e->call_stmt_cannot_inline_p)
+	    {
+	      if (dump_file)
 		fprintf (dump_file, "Not inlining: %s.\n",
 			 cgraph_inline_failed_string (e->inline_failed));
-	      }
-	    continue;
-	  }
-	if (!e->callee->analyzed)
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
+	      continue;
+	    }
+	  if (!e->callee->analyzed)
+	    {
+	      if (dump_file)
 		fprintf (dump_file,
 			 "Not inlining: Function body no longer available.\n");
-	      }
-	    continue;
-	  }
-	if (!tree_can_inline_p (e))
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
+	      continue;
+	    }
+	  if (!tree_can_inline_p (e))
+	    {
+	      if (dump_file)
 		fprintf (dump_file,
 			 "Not inlining: %s.",
-                         cgraph_inline_failed_string (e->inline_failed));
-	      }
-	    continue;
-	  }
-	if (cgraph_default_inline_p (e->callee, &failed_reason))
-	  inlined |= try_inline (e, mode, depth);
-      }
-  node->aux = (void *)(size_t) old_mode;
+			 cgraph_inline_failed_string (e->inline_failed));
+	      continue;
+	    }
+	  if (cgraph_default_inline_p (e->callee, &failed_reason))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, " Inlining %s into %s.\n",
+			 cgraph_node_name (e->callee),
+			 cgraph_node_name (e->caller));
+	      cgraph_mark_inline (e);
+	      inlined = true;
+	    }
+	}
+      BITMAP_FREE (visited);
+    }
   return inlined;
 }
 
@@ -1620,27 +1664,51 @@
 
   if (sorrycount || errorcount)
     return 0;
-  while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
-         && cgraph_decide_inlining_incrementally (node,
-  					          iterations
-					          ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0))
+
+  if (!optimize
+      || flag_no_inline
+      || !flag_early_inlining)
     {
+      /* When not optimizing or not inlining inline only always-inline
+	 functions.  */
+      cgraph_decide_inlining_incrementally (node, INLINE_ALWAYS_INLINE);
       timevar_push (TV_INTEGRATION);
       todo |= optimize_inline_calls (current_function_decl);
-      iterations++;
       timevar_pop (TV_INTEGRATION);
     }
-  if (dump_file)
-    fprintf (dump_file, "Iterations: %i\n", iterations);
-  cfun->always_inline_functions_inlined = true;
-  return todo;
-}
+  else
+    {
+      if (lookup_attribute ("flatten",
+			    DECL_ATTRIBUTES (node->decl)) != NULL)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Flattening %s\n", cgraph_node_name (node));
+	  cgraph_flatten (node);
+	  timevar_push (TV_INTEGRATION);
+	  todo |= optimize_inline_calls (current_function_decl);
+	  timevar_pop (TV_INTEGRATION);
+	}
+      /* We iterate incremental inlining to get trivial cases of indirect
+	 inlining.  */
+      while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+	     && cgraph_decide_inlining_incrementally (node,
+						      iterations
+						      ? INLINE_SIZE_NORECURSIVE
+						      : INLINE_SIZE))
+	{
+	  timevar_push (TV_INTEGRATION);
+	  todo |= optimize_inline_calls (current_function_decl);
+	  iterations++;
+	  timevar_pop (TV_INTEGRATION);
+	}
+      if (dump_file)
+	fprintf (dump_file, "Iterations: %i\n", iterations);
+    }
 
-/* When inlining shall be performed.  */
-static bool
-cgraph_gate_early_inlining (void)
-{
-  return flag_early_inlining;
+  cfun->always_inline_functions_inlined = true;
+
+  return todo;
 }
 
 struct gimple_opt_pass pass_early_inline =
@@ -1648,7 +1716,7 @@
  {
   GIMPLE_PASS,
   "einline",	 			/* name */
-  cgraph_gate_early_inlining,		/* gate */
+  NULL,					/* gate */
   cgraph_early_inlining,		/* execute */
   NULL,					/* sub */
   NULL,					/* next */
@@ -1859,10 +1927,10 @@
   node->global.stack_frame_offset = 0;
 
   /* Can this function be inlined at all?  */
-  node->local.inlinable = tree_inlinable_function_p (current_function_decl);
+  node->local.inlinable = tree_inlinable_function_p (node->decl);
   if (node->local.inlinable && !node->local.disregard_inline_limits)
     node->local.disregard_inline_limits
-      = DECL_DISREGARD_INLINE_LIMITS (current_function_decl);
+      = DECL_DISREGARD_INLINE_LIMITS (node->decl);
   estimate_function_body_sizes (node);
   /* Inlining characteristics are maintained by the cgraph_mark_inline.  */
   node->global.time = inline_summary (node)->self_time;
@@ -1904,21 +1972,10 @@
 static void
 inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
 {
-  struct cgraph_edge *cs;
-
-  if (!flag_ipa_cp)
-    {
-      ipa_initialize_node_params (node);
-      ipa_detect_param_modifications (node);
-    }
+  ipa_initialize_node_params (node);
+  ipa_detect_param_modifications (node);
   ipa_analyze_params_uses (node);
-
-  if (!flag_ipa_cp)
-    for (cs = node->callees; cs; cs = cs->next_callee)
-      {
-	ipa_count_arguments (cs);
-	ipa_compute_jump_functions (cs);
-      }
+  ipa_compute_jump_functions (node);
 
   if (dump_file)
     {
@@ -2026,18 +2083,33 @@
    active, we don't need to write them twice.  */
 
 static void
-inline_write_summary (cgraph_node_set set)
+inline_write_summary (cgraph_node_set set,
+		      varpool_node_set vset ATTRIBUTE_UNUSED)
 {
   if (flag_indirect_inlining && !flag_ipa_cp)
     ipa_prop_write_jump_functions (set);
 }
 
+/* When to run IPA inlining.  Inlining of always-inline functions
+   happens during early inlining.  */
+
+static bool
+gate_cgraph_decide_inlining (void)
+{
+  /* ???  We'd like to skip this if not optimizing or not inlining as
+     all always-inline functions have been processed by early
+     inlining already.  But this at least breaks EH with C++ as
+     we need to unconditionally run fixup_cfg even at -O0.
+     So leave it on unconditionally for now.  */
+  return 1;
+}
+
 struct ipa_opt_pass_d pass_ipa_inline =
 {
  {
   IPA_PASS,
   "inline",				/* name */
-  NULL,					/* gate */
+  gate_cgraph_decide_inlining,		/* gate */
   cgraph_decide_inlining,		/* execute */
   NULL,					/* sub */
   NULL,					/* next */
@@ -2048,13 +2120,14 @@
   0,					/* properties_destroyed */
   TODO_remove_functions,		/* todo_flags_finish */
   TODO_dump_cgraph | TODO_dump_func
-  | TODO_remove_functions		/* todo_flags_finish */
+  | TODO_remove_functions | TODO_ggc_collect	/* todo_flags_finish */
  },
  inline_generate_summary,		/* generate_summary */
  inline_write_summary,			/* write_summary */
  inline_read_summary,			/* read_summary */
- NULL,					/* function_read_summary */
- lto_ipa_fixup_call_notes,		/* stmt_fixup */
+ NULL,					/* write_optimization_summary */
+ NULL,					/* read_optimization_summary */
+ NULL,					/* stmt_fixup */
  0,					/* TODOs */
  inline_transform,			/* function_transform */
  NULL,					/* variable_transform */