CbC/CbC_gcc: gcc/ipa-inline.c comparison

comparison gcc/ipa-inline.c @ 63:b7f97abdc517 gcc-4.6-20100522

update gcc from gcc-4.5.0 to gcc-4.6

author	ryoma <e075725@ie.u-ryukyu.ac.jp>
date	Mon, 24 May 2010 12:47:05 +0900
parents	77e2b8dfacca
children	f6334be47118

comparison

equal deleted inserted replaced

-:3c8a44c06a95
+:b7f97abdc517
 /* Inlining decision heuristics.
-Copyright (C) 2003, 2004, 2007, 2008, 2009 Free Software Foundation, Inc.
+Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010
+Free Software Foundation, Inc.
 Contributed by Jan Hubicka
 This file is part of GCC.
 GCC is free software; you can redistribute it and/or modify it under
 #include "tree-inline.h"
 #include "langhooks.h"
 #include "flags.h"
 #include "cgraph.h"
 #include "diagnostic.h"
+#include "gimple-pretty-print.h"
 #include "timevar.h"
 #include "params.h"
 #include "fibheap.h"
 #include "intl.h"
 #include "tree-pass.h"
 INLINE_ALWAYS_INLINE,
 INLINE_SIZE_NORECURSIVE,
 INLINE_SIZE,
 INLINE_ALL
 };
 static bool
-cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode,
+cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode);
-				      int);
+static void cgraph_flatten (struct cgraph_node *node);
 /* Statistics we collect about inlining algorithm.  */
 static int ncalls_inlined;
 static int nfunctions_inlined;
 update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
 	}
 else
 	{
 	  struct cgraph_node *n;
-	  n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+	  n = cgraph_clone_node (e->callee, e->callee->decl,
+				 e->count, e->frequency, e->loop_nest,
 				 update_original, NULL);
 	  cgraph_redirect_edge_callee (e, n);
 	}
 }
 + inline_summary (e->caller)->estimated_self_stack_size;
 peak = e->callee->global.stack_frame_offset
 + inline_summary (e->callee)->estimated_self_stack_size;
 if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
 e->callee->global.inlined_to->global.estimated_stack_size = peak;
+cgraph_propagate_frequency (e->callee);
 /* Recursively clone all bodies.  */
 for (e = e->callee->callees; e; e = e->next_callee)
 if (!e->inline_failed)
 cgraph_clone_inlined_nodes (e, duplicate, update_original);
 {
 int old_size = 0, new_size = 0;
 struct cgraph_node *to = NULL, *what;
 struct cgraph_edge *curr = e;
 int freq;
-bool duplicate = false;
-int orig_size = e->callee->global.size;
 gcc_assert (e->inline_failed);
 e->inline_failed = CIF_OK;
+DECL_POSSIBLY_INLINED (e->callee->decl) = true;
-if (!e->callee->global.inlined)
-DECL_POSSIBLY_INLINED (e->callee->decl) = true;
-e->callee->global.inlined = true;
-if (e->callee->callers->next_caller
-|| !cgraph_can_remove_if_no_direct_calls_p (e->callee)
-|| e->callee->same_comdat_group)
-duplicate = true;
 cgraph_clone_inlined_nodes (e, true, update_original);
 what = e->callee;
 freq = e->frequency;
 to->global.time = cgraph_estimate_time_after_inlining (freq, to, what);
 }
 gcc_assert (what->global.inlined_to == to);
 if (new_size > old_size)
 overall_size += new_size - old_size;
-if (!duplicate)
-overall_size -= orig_size;
 ncalls_inlined++;
 if (flag_indirect_inlining)
 return ipa_propagate_indirect_call_infos (curr, new_edges);
 else
 return false;
 }
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
+/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.  */
-Return following unredirected edge in the list of callers
-of EDGE->CALLEE  */
+static void
-static struct cgraph_edge *
 cgraph_mark_inline (struct cgraph_edge *edge)
 {
 struct cgraph_node *to = edge->caller;
 struct cgraph_node *what = edge->callee;
 struct cgraph_edge *e, *next;
 cgraph_mark_inline_edge (e, true, NULL);
 	  if (e == edge)
 	    edge = next;
 	}
 }
-return edge;
 }
 /* Estimate the growth caused by inlining NODE into all callees.  */
 static int
 static bool
 cgraph_default_inline_p (struct cgraph_node *n, cgraph_inline_failed_t *reason)
 {
 tree decl = n->decl;
+if (n->local.disregard_inline_limits)
+return true;
 if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl))
 {
 if (reason)
 	*reason = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
 the costs of all caller edges of nodes affected are recomputed so the
 metrics may accurately depend on values such as number of inlinable callers
 of the function or function body size.  */
 static int
-cgraph_edge_badness (struct cgraph_edge *edge)
+cgraph_edge_badness (struct cgraph_edge *edge, bool dump)
 {
 gcov_type badness;
 int growth =
-cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+(cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+- edge->caller->global.size);
-growth -= edge->caller->global.size;
+if (edge->callee->local.disregard_inline_limits)
+return INT_MIN;
+if (dump)
+{
+fprintf (dump_file, "    Badness calculcation for %s -> %s\n",
+	       cgraph_node_name (edge->caller),
+	       cgraph_node_name (edge->callee));
+fprintf (dump_file, "      growth %i, time %i-%i, size %i-%i\n",
+	       growth,
+	       edge->callee->global.time,
+	       inline_summary (edge->callee)->time_inlining_benefit,
+	       edge->callee->global.size,
+	       inline_summary (edge->callee)->size_inlining_benefit);
+}
 /* Always prefer inlining saving code size.  */
 if (growth <= 0)
-badness = INT_MIN - growth;
+{
+badness = INT_MIN - growth;
+if (dump)
+	fprintf (dump_file, "      %i: Growth %i < 0\n", (int) badness,
+		 growth);
+}
 /* When profiling is available, base priorities -(#calls / growth).
 So we optimize for overall number of "executed" inlined calls.  */
 else if (max_count)
-badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1))
+{
-	      * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+badness =
+	((int)
+	 ((double) edge->count * INT_MIN / max_count / (max_benefit + 1)) *
+	 (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+if (dump)
+	{
+	  fprintf (dump_file,
+		   "      %i (relative %f): profile info. Relative count %f"
+		   " * Relative benefit %f\n",
+		   (int) badness, (double) badness / INT_MIN,
+		   (double) edge->count / max_count,
+		   (double) (inline_summary (edge->callee)->
+			     time_inlining_benefit + 1) / (max_benefit + 1));
+	}
+}
 /* When function local profile is available, base priorities on
 growth / frequency, so we optimize for overall frequency of inlined
 calls.  This is not too accurate since while the call might be frequent
 within function, the function itself is infrequent.
 priorities slightly in this direction (so fewer times called functions
 of the same size gets priority).  */
 else if (flag_guess_branch_prob)
 {
 int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1;
+int benefitperc;
+int growth_for_all;
 badness = growth * 10000;
-div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
+benefitperc =
-	          / (edge->callee->global.time + 1) + 1, 100);
+	MIN (100 * inline_summary (edge->callee)->time_inlining_benefit /
+	     (edge->callee->global.time + 1) +1, 100);
+div *= benefitperc;
 /* Decrease badness if call is nested.  */
 /* Compress the range so we don't overflow.  */
 if (div > 10000)
 	div = 10000 + ceil_log2 (div) - 8;
 if (div < 1)
 	div = 1;
 if (badness > 0)
 	badness /= div;
-badness += cgraph_estimate_growth (edge->callee);
+growth_for_all = cgraph_estimate_growth (edge->callee);
+badness += growth_for_all;
 if (badness > INT_MAX)
-badness = INT_MAX;
+	badness = INT_MAX;
+if (dump)
+	{
+	  fprintf (dump_file,
+		   "      %i: guessed profile. frequency %i, overall growth %i,"
+		   " benefit %i%%, divisor %i\n",
+		   (int) badness, edge->frequency, growth_for_all, benefitperc, div);
+	}
 }
 /* When function local profile is not available or it does not give
 useful information (ie frequency is zero), base the cost on
 loop nest and overall size growth, so we optimize for overall number
 of functions fully inlined in program.  */
 /* Decrease badness if call is nested.  */
 if (badness > 0)
 	badness >>= nest;
 else
-{
+	{
 	  badness <<= nest;
-}
+	}
-}
+if (dump)
+	fprintf (dump_file, "      %i: no profile. nest %i\n", (int) badness,
+		 nest);
+}
+/* Ensure that we did not overflow in all the fixed point math above.  */
+gcc_assert (badness >= INT_MIN);
+gcc_assert (badness <= INT_MAX - 1);
 /* Make recursive inlining happen always after other inlining is done.  */
 if (cgraph_recursive_inlining_p (edge->caller, edge->callee, NULL))
 return badness + 1;
 else
 return badness;
 		    bitmap updated_nodes)
 {
 struct cgraph_edge *edge;
 cgraph_inline_failed_t failed_reason;
-if (!node->local.inlinable || node->local.disregard_inline_limits
+if (!node->local.inlinable
 || node->global.inlined_to)
 return;
 if (bitmap_bit_p (updated_nodes, node->uid))
 return;
 bitmap_set_bit (updated_nodes, node->uid);
 }
 for (edge = node->callers; edge; edge = edge->next_caller)
 if (edge->inline_failed)
 {
-	int badness = cgraph_edge_badness (edge);
+	int badness = cgraph_edge_badness (edge, false);
 	if (edge->aux)
 	  {
 	    fibnode_t n = (fibnode_t) edge->aux;
 	    gcc_assert (n->data == edge);
 	    if (n->key == badness)
 	      continue;
 	    /* fibheap_replace_key only increase the keys.  */
-	    if (fibheap_replace_key (heap, n, badness))
+	    if (badness < n->key)
-	      continue;
+	      {
+		fibheap_replace_key (heap, n, badness);
+		gcc_assert (n->key == badness);
+	        continue;
+	      }
 	    fibheap_delete_node (heap, (fibnode_t) edge->aux);
 	  }
 	edge->aux = fibheap_insert (heap, badness, edge);
 }
 }
 struct cgraph_edge *e;
 struct cgraph_node *master_clone, *next;
 int depth = 0;
 int n = 0;
+/* It does not make sense to recursively inline always-inline functions
+as we are going to sorry() on the remaining calls anyway.  */
+if (node->local.disregard_inline_limits
+&& lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)))
+return false;
 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))
 || (!flag_inline_functions && !DECL_DECLARED_INLINE_P (node->decl)))
 return false;
 if (DECL_DECLARED_INLINE_P (node->decl))
 fprintf (dump_file,
 	     "  Performing recursive inlining on %s\n",
 	     cgraph_node_name (node));
 /* We need original clone to copy around.  */
-master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1,
+master_clone = cgraph_clone_node (node, node->decl,
+				    node->count, CGRAPH_FREQ_BASE, 1,
 				    false, NULL);
 master_clone->needed = true;
 for (e = master_clone->callees; e; e = e->next_callee)
 if (!e->inline_failed)
 cgraph_clone_inlined_nodes (e, true, false);
 while (VEC_length (cgraph_edge_p, new_edges) > 0)
 {
 struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
 gcc_assert (!edge->aux);
-edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
 }
 }
 /* We use greedy algorithm for inlining of small functions:
 /* Put all inline candidates into the heap.  */
 for (node = cgraph_nodes; node; node = node->next)
 {
-if (!node->local.inlinable || !node->callers
+if (!node->local.inlinable || !node->callers)
-	  || node->local.disregard_inline_limits)
 	continue;
 if (dump_file)
 	fprintf (dump_file, "Considering inline candidate %s.\n", cgraph_node_name (node));
 node->global.estimated_growth = INT_MIN;
 for (edge = node->callers; edge; edge = edge->next_caller)
 	if (edge->inline_failed)
 	  {
 	    gcc_assert (!edge->aux);
-	    edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+	    edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
 	  }
 }
 max_size = compute_max_insns (overall_size);
 min_size = overall_size;
 while (overall_size <= max_size
-	 && (edge = (struct cgraph_edge *) fibheap_extract_min (heap)))
+	 && !fibheap_empty (heap))
 {
 int old_size = overall_size;
-struct cgraph_node *where;
+struct cgraph_node *where, *callee;
-int growth =
+int badness = fibheap_min_key (heap);
-	cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+int growth;
 cgraph_inline_failed_t not_good = CIF_OK;
-growth -= edge->caller->global.size;
+edge = (struct cgraph_edge *) fibheap_extract_min (heap);
+gcc_assert (edge->aux);
+edge->aux = NULL;
+if (!edge->inline_failed)
+	continue;
+#ifdef ENABLE_CHECKING
+gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+callee = edge->callee;
+growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+		- edge->caller->global.size);
 if (dump_file)
 	{
 	  fprintf (dump_file,
 		   "\nConsidering %s with %i size\n",
 	  fprintf (dump_file,
 		   " to be inlined into %s in %s:%i\n"
 		   " Estimated growth after inlined into all callees is %+i insns.\n"
 		   " Estimated badness is %i, frequency %.2f.\n",
 		   cgraph_node_name (edge->caller),
-		   gimple_filename ((const_gimple) edge->call_stmt),
+		   flag_wpa ? "unknown"
-		   gimple_lineno ((const_gimple) edge->call_stmt),
+		   : gimple_filename ((const_gimple) edge->call_stmt),
+		   flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
 		   cgraph_estimate_growth (edge->callee),
-		   cgraph_edge_badness (edge),
+		   badness,
 		   edge->frequency / (double)CGRAPH_FREQ_BASE);
 	  if (edge->count)
 	    fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
-	}
+	  if (dump_flags & TDF_DETAILS)
-gcc_assert (edge->aux);
+	    cgraph_edge_badness (edge, true);
-edge->aux = NULL;
+	}
-if (!edge->inline_failed)
-	continue;
 /* When not having profile info ready we don't weight by any way the
 position of call in procedure itself.  This means if call of
 	 function A from function B seems profitable to inline, the recursive
 	 call of function A in inline copy of A in B will look profitable too
 		fprintf (dump_file, " inline_failed:Recursive inlining performed only for function itself.\n");
 	      continue;
 	    }
 	}
-if (!cgraph_maybe_hot_edge_p (edge))
+if (edge->callee->local.disregard_inline_limits)
+	;
+else if (!cgraph_maybe_hot_edge_p (edge))
 	not_good = CIF_UNLIKELY_CALL;
-if (!flag_inline_functions
+else if (!flag_inline_functions
 	  && !DECL_DECLARED_INLINE_P (edge->callee->decl))
 	not_good = CIF_NOT_DECLARED_INLINED;
-if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
+else if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
 	not_good = CIF_OPTIMIZING_FOR_SIZE;
 if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0)
 	{
 if (!cgraph_recursive_inlining_p (edge->caller, edge->callee,
 				            &edge->inline_failed))
 	 After inlining these properties might change for the function we
 	 inlined into (since it's body size changed) and for the functions
 	 called by function we inlined (since number of it inlinable callers
 	 might change).  */
 update_caller_keys (heap, where, updated_nodes);
+/* We removed one call of the function we just inlined.  If offline
+	 copy is still needed, be sure to update the keys.  */
+if (callee != where && !callee->global.inlined_to)
+update_caller_keys (heap, callee, updated_nodes);
 bitmap_clear (updated_nodes);
 if (dump_file)
 	{
 	  fprintf (dump_file,
 	  if (dump_file)
 	    fprintf (dump_file, "New minimal size reached: %i\n", min_size);
 	}
 }
-while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL)
+while (!fibheap_empty (heap))
 {
+int badness = fibheap_min_key (heap);
+edge = (struct cgraph_edge *) fibheap_extract_min (heap);
 gcc_assert (edge->aux);
 edge->aux = NULL;
+if (!edge->inline_failed)
+	continue;
+#ifdef ENABLE_CHECKING
+gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+if (dump_file)
+	{
+	  fprintf (dump_file,
+		   "\nSkipping %s with %i size\n",
+		   cgraph_node_name (edge->callee),
+		   edge->callee->global.size);
+	  fprintf (dump_file,
+		   " called by %s in %s:%i\n"
+		   " Estimated growth after inlined into all callees is %+i insns.\n"
+		   " Estimated badness is %i, frequency %.2f.\n",
+		   cgraph_node_name (edge->caller),
+		   flag_wpa ? "unknown"
+		   : gimple_filename ((const_gimple) edge->call_stmt),
+		   flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
+		   cgraph_estimate_growth (edge->callee),
+		   badness,
+		   edge->frequency / (double)CGRAPH_FREQ_BASE);
+	  if (edge->count)
+	    fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+	  if (dump_flags & TDF_DETAILS)
+	    cgraph_edge_badness (edge, true);
+	}
 if (!edge->callee->local.disregard_inline_limits && edge->inline_failed
 && !cgraph_recursive_inlining_p (edge->caller, edge->callee,
 				           &edge->inline_failed))
 	edge->inline_failed = CIF_INLINE_UNIT_GROWTH_LIMIT;
 }
 if (new_indirect_edges)
 VEC_free (cgraph_edge_p, heap, new_indirect_edges);
 fibheap_delete (heap);
 BITMAP_FREE (updated_nodes);
+}
+/* Flatten NODE from the IPA inliner.  */
+static void
+cgraph_flatten (struct cgraph_node *node)
+{
+struct cgraph_edge *e;
+/* We shouldn't be called recursively when we are being processed.  */
+gcc_assert (node->aux == NULL);
+node->aux = (void *)(size_t) INLINE_ALL;
+for (e = node->callees; e; e = e->next_callee)
+{
+struct cgraph_node *orig_callee;
+if (e->call_stmt_cannot_inline_p)
+	continue;
+if (!e->callee->analyzed)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Not inlining: Function body not available.\n");
+	  continue;
+	}
+/* We've hit cycle?  It is time to give up.  */
+if (e->callee->aux)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Not inlining %s into %s to avoid cycle.\n",
+		     cgraph_node_name (e->callee),
+		     cgraph_node_name (e->caller));
+	  e->inline_failed = CIF_RECURSIVE_INLINING;
+	  continue;
+	}
+/* When the edge is already inlined, we just need to recurse into
+	 it in order to fully flatten the leaves.  */
+if (!e->inline_failed)
+	{
+	  cgraph_flatten (e->callee);
+	  continue;
+	}
+if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Not inlining: recursive call.\n");
+	  continue;
+	}
+if (!tree_can_inline_p (e))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Not inlining: %s",
+		     cgraph_inline_failed_string (e->inline_failed));
+	  continue;
+	}
+/* Inline the edge and flatten the inline clone.  Avoid
+recursing through the original node if the node was cloned.  */
+if (dump_file)
+	fprintf (dump_file, " Inlining %s into %s.\n",
+		 cgraph_node_name (e->callee),
+		 cgraph_node_name (e->caller));
+orig_callee = e->callee;
+cgraph_mark_inline_edge (e, true, NULL);
+if (e->callee != orig_callee)
+	orig_callee->aux = (void *)(size_t) INLINE_ALL;
+cgraph_flatten (e->callee);
+if (e->callee != orig_callee)
+	orig_callee->aux = NULL;
+}
+node->aux = NULL;
 }
 /* Decide on the inlining.  We do so in the topological order to avoid
 expenses on updating data structures.  */
 int nnodes;
 struct cgraph_node **order =
 XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
 int old_size = 0;
 int i;
-bool redo_always_inline = true;
 int initial_size = 0;
 cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
 if (in_lto_p && flag_indirect_inlining)
 ipa_update_after_lto_read ();
+if (flag_indirect_inlining)
+ipa_create_all_structures_for_iinln ();
 max_count = 0;
 max_benefit = 0;
 for (node = cgraph_nodes; node; node = node->next)
 if (node->analyzed)
 for (node = cgraph_nodes; node; node = node->next)
 node->aux = 0;
 if (dump_file)
-fprintf (dump_file, "\nInlining always_inline functions:\n");
+fprintf (dump_file, "\nFlattening functions:\n");
-/* In the first pass mark all always_inline edges.  Do this with a priority
+/* In the first pass handle functions to be flattened.  Do this with
-so none of our later choices will make this impossible.  */
+a priority so none of our later choices will make this impossible.  */
-while (redo_always_inline)
+for (i = nnodes - 1; i >= 0; i--)
 {
-redo_always_inline = false;
+node = order[i];
-for (i = nnodes - 1; i >= 0; i--)
-	{
+/* Handle nodes to be flattened, but don't update overall unit
-	  struct cgraph_edge *e, *next;
+	 size.  Calling the incremental inliner here is lame,
+	 a simple worklist should be enough.  What should be left
-	  node = order[i];
+	 here from the early inliner (if it runs) is cyclic cases.
+	 Ideally when processing callees we stop inlining at the
-	  /* Handle nodes to be flattened, but don't update overall unit
+	 entry of cycles, possibly cloning that entry point and
-	     size.  */
+	 try to flatten itself turning it into a self-recursive
-	  if (lookup_attribute ("flatten",
+	 function.  */
-				DECL_ATTRIBUTES (node->decl)) != NULL)
+if (lookup_attribute ("flatten",
-	    {
+			    DECL_ATTRIBUTES (node->decl)) != NULL)
-	      if (dump_file)
+	{
-		fprintf (dump_file,
-			 "Flattening %s\n", cgraph_node_name (node));
-	      cgraph_decide_inlining_incrementally (node, INLINE_ALL, 0);
-	    }
-	  if (!node->local.disregard_inline_limits)
-	    continue;
 	  if (dump_file)
 	    fprintf (dump_file,
-		     "\nConsidering %s size:%i (always inline)\n",
+		     "Flattening %s\n", cgraph_node_name (node));
-		     cgraph_node_name (node), node->global.size);
+	  cgraph_flatten (node);
-	  old_size = overall_size;
-	  for (e = node->callers; e; e = next)
-	    {
-	      next = e->next_caller;
-	      if (!e->inline_failed || e->call_stmt_cannot_inline_p)
-		continue;
-	      if (cgraph_recursive_inlining_p (e->caller, e->callee,
-					       &e->inline_failed))
-		continue;
-	      if (!tree_can_inline_p (e))
-continue;
-	      if (cgraph_mark_inline_edge (e, true, NULL))
-		redo_always_inline = true;
-	      if (dump_file)
-		fprintf (dump_file,
-			 " Inlined into %s which now has size %i.\n",
-			 cgraph_node_name (e->caller),
-			 e->caller->global.size);
-	    }
-	  /* Inlining self recursive function might introduce new calls to
-	     themselves we didn't see in the loop above.  Fill in the proper
-	     reason why inline failed.  */
-	  for (e = node->callers; e; e = e->next_caller)
-	    if (e->inline_failed)
-	      e->inline_failed = CIF_RECURSIVE_INLINING;
-	  if (dump_file)
-	    fprintf (dump_file,
-		     " Inlined for a net change of %+i size.\n",
-		     overall_size - old_size);
 	}
 }
 cgraph_decide_inlining_of_small_functions ();
 		}
 	      if (cgraph_check_inline_limits (node->callers->caller, node,
 					      &reason, false))
 		{
+		  struct cgraph_node *caller = node->callers->caller;
 		  cgraph_mark_inline (node->callers);
 		  if (dump_file)
 		    fprintf (dump_file,
 			     " Inlined into %s which now has %i size"
 			     " for a net change of %+i size.\n",
-			     cgraph_node_name (node->callers->caller),
+			     cgraph_node_name (caller),
-			     node->callers->caller->global.size,
+			     caller->global.size,
 			     overall_size - old_size);
 		}
 	      else
 		{
 		  if (dump_file)
 	}
 }
 /* Free ipa-prop structures if they are no longer needed.  */
 if (flag_indirect_inlining)
-free_all_ipa_structures_after_iinln ();
+ipa_free_all_structures_after_iinln ();
 if (dump_file)
 fprintf (dump_file,
 	     "\nInlined %i calls, eliminated %i functions, "
 	     "size %i turned to %i size.\n\n",
 	     overall_size);
 free (order);
 return 0;
 }
-/* Try to inline edge E from incremental inliner.  MODE specifies mode
-of inliner.
-We are detecting cycles by storing mode of inliner into cgraph_node last
-time we visited it in the recursion.  In general when mode is set, we have
-recursive inlining, but as an special case, we want to try harder inline
-ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
-flatten, b being always inline.  Flattening 'a' will collapse
-a->b->c before hitting cycle.  To accommodate always inline, we however
-need to inline a->b->c->b.
-So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
-stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode.  */
-static bool
-try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth)
-{
-struct cgraph_node *callee = e->callee;
-enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux;
-bool always_inline = e->callee->local.disregard_inline_limits;
-bool inlined = false;
-/* We've hit cycle?  */
-if (callee_mode)
-{
-/* It is first time we see it and we are not in ALWAY_INLINE only
-	 mode yet.  and the function in question is always_inline.  */
-if (always_inline && mode != INLINE_ALWAYS_INLINE)
-	{
-	  if (dump_file)
-	    {
-	      indent_to (dump_file, depth);
-	      fprintf (dump_file,
-		       "Hit cycle in %s, switching to always inline only.\n",
-		       cgraph_node_name (callee));
-	    }
-	  mode = INLINE_ALWAYS_INLINE;
-	}
-/* Otherwise it is time to give up.  */
-else
-	{
-	  if (dump_file)
-	    {
-	      indent_to (dump_file, depth);
-	      fprintf (dump_file,
-		       "Not inlining %s into %s to avoid cycle.\n",
-		       cgraph_node_name (callee),
-		       cgraph_node_name (e->caller));
-	    }
-	  e->inline_failed = (e->callee->local.disregard_inline_limits
-		              ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
-return false;
-	}
-}
-callee->aux = (void *)(size_t) mode;
-if (dump_file)
-{
-indent_to (dump_file, depth);
-fprintf (dump_file, " Inlining %s into %s.\n",
-	       cgraph_node_name (e->callee),
-	       cgraph_node_name (e->caller));
-}
-if (e->inline_failed)
-{
-cgraph_mark_inline (e);
-/* In order to fully inline always_inline functions, we need to
-	 recurse here, since the inlined functions might not be processed by
-	 incremental inlining at all yet.
-	 Also flattening needs to be done recursively.  */
-if (mode == INLINE_ALL || always_inline)
-	cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1);
-inlined = true;
-}
-callee->aux = (void *)(size_t) callee_mode;
-return inlined;
-}
 /* Return true when N is leaf function.  Accept cheap (pure&const) builtins
 in leaf functions.  */
 static bool
 leaf_node_p (struct cgraph_node *n)
 {
 return false;
 return true;
 }
 /* Decide on the inlining.  We do so in the topological order to avoid
-expenses on updating data structures.
+expenses on updating data structures.  */
-DEPTH is depth of recursion, used only for debug output.  */
 static bool
 cgraph_decide_inlining_incrementally (struct cgraph_node *node,
-				      enum inlining_mode mode,
+				      enum inlining_mode mode)
-				      int depth)
 {
 struct cgraph_edge *e;
 bool inlined = false;
 cgraph_inline_failed_t failed_reason;
-enum inlining_mode old_mode;
 #ifdef ENABLE_CHECKING
 verify_cgraph_node (node);
 #endif
-old_mode = (enum inlining_mode) (size_t)node->aux;
 if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE
 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL)
 {
 if (dump_file)
-	{
+	fprintf (dump_file, "Incrementally flattening %s\n",
-	  indent_to (dump_file, depth);
+		 cgraph_node_name (node));
-	  fprintf (dump_file, "Flattening %s\n", cgraph_node_name (node));
-	}
 mode = INLINE_ALL;
 }
-node->aux = (void *)(size_t) mode;
 /* First of all look for always inline functions.  */
 if (mode != INLINE_SIZE_NORECURSIVE)
 for (e = node->callees; e; e = e->next_callee)
 {
 	if (!e->callee->local.disregard_inline_limits
 	    && (mode != INLINE_ALL || !e->callee->local.inlinable))
 	  continue;
 	if (e->call_stmt_cannot_inline_p)
 	  continue;
-	/* When the edge is already inlined, we just need to recurse into
-	   it in order to fully flatten the leaves.  */
-	if (!e->inline_failed && mode == INLINE_ALL)
-	  {
-	    inlined |= try_inline (e, mode, depth);
-	    continue;
-	  }
 	if (dump_file)
-	  {
+	  fprintf (dump_file,
-	    indent_to (dump_file, depth);
+		   "Considering to always inline inline candidate %s.\n",
-	    fprintf (dump_file,
+		   cgraph_node_name (e->callee));
-		     "Considering to always inline inline candidate %s.\n",
-		     cgraph_node_name (e->callee));
-	  }
 	if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
 	  {
 	    if (dump_file)
-	      {
+	      fprintf (dump_file, "Not inlining: recursive call.\n");
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: recursive call.\n");
-	      }
 	    continue;
 	  }
 	if (!tree_can_inline_p (e))
 	  {
 	    if (dump_file)
-	      {
+	      fprintf (dump_file,
-		indent_to (dump_file, depth);
+		       "Not inlining: %s",
-		fprintf (dump_file,
+		       cgraph_inline_failed_string (e->inline_failed));
-			 "Not inlining: %s",
-cgraph_inline_failed_string (e->inline_failed));
-	      }
 	    continue;
 	  }
 	if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
 	    != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
 	  {
 	    if (dump_file)
-	      {
+	      fprintf (dump_file, "Not inlining: SSA form does not match.\n");
-		indent_to (dump_file, depth);
-		fprintf (dump_file, "Not inlining: SSA form does not match.\n");
-	      }
 	    continue;
 	  }
 	if (!e->callee->analyzed)
 	  {
 	    if (dump_file)
-	      {
+	      fprintf (dump_file,
-		indent_to (dump_file, depth);
+		       "Not inlining: Function body no longer available.\n");
-		fprintf (dump_file,
-			 "Not inlining: Function body no longer available.\n");
-	      }
 	    continue;
 	  }
-	inlined |= try_inline (e, mode, depth);
+	if (dump_file)
+	  fprintf (dump_file, " Inlining %s into %s.\n",
+		   cgraph_node_name (e->callee),
+		   cgraph_node_name (e->caller));
+	cgraph_mark_inline (e);
+	inlined = true;
 }
 /* Now do the automatic inlining.  */
-if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE)
+if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE
-for (e = node->callees; e; e = e->next_callee)
+/* Never inline regular functions into always-inline functions
-{
+	 during incremental inlining.  */
-int allowed_growth = 0;
+&& !node->local.disregard_inline_limits)
-	if (!e->callee->local.inlinable
+{
-	    || !e->inline_failed
+bitmap visited = BITMAP_ALLOC (NULL);
-	    || e->callee->local.disregard_inline_limits)
+for (e = node->callees; e; e = e->next_callee)
-	  continue;
+	{
-	if (dump_file)
+	  int allowed_growth = 0;
-	  fprintf (dump_file, "Considering inline candidate %s.\n",
+	  if (!e->callee->local.inlinable
-		   cgraph_node_name (e->callee));
+	      || !e->inline_failed
-	if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+	      || e->callee->local.disregard_inline_limits)
-	  {
+	    continue;
-	    if (dump_file)
+	  /* We are inlining a function to all call-sites in node
-	      {
+	     or to none.  So visit each candidate only once.  */
-		indent_to (dump_file, depth);
+	  if (!bitmap_set_bit (visited, e->callee->uid))
+	    continue;
+	  if (dump_file)
+	    fprintf (dump_file, "Considering inline candidate %s.\n",
+		     cgraph_node_name (e->callee));
+	  if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+	    {
+	      if (dump_file)
 		fprintf (dump_file, "Not inlining: recursive call.\n");
-	      }
+	      continue;
-	    continue;
+	    }
-	  }
+	  if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
-	if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+	      != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
-	    != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+	    {
-	  {
+	      if (dump_file)
-	    if (dump_file)
+		fprintf (dump_file,
-	      {
+			 "Not inlining: SSA form does not match.\n");
-		indent_to (dump_file, depth);
+	      continue;
-		fprintf (dump_file, "Not inlining: SSA form does not match.\n");
+	    }
-	      }
-	    continue;
+	  if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
-	  }
+	      && optimize_function_for_speed_p (cfun))
+	    allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
-	if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
-	    && optimize_function_for_speed_p (cfun))
+	  /* When the function body would grow and inlining the function
-	  allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+	     won't eliminate the need for offline copy of the function,
+	     don't inline.  */
-	/* When the function body would grow and inlining the function won't
+	  if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
-	   eliminate the need for offline copy of the function, don't inline.
+	       || (!flag_inline_functions
-	 */
+		   && !DECL_DECLARED_INLINE_P (e->callee->decl)))
-	if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
+	      && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
-	     || (!flag_inline_functions
+		  > e->caller->global.size + allowed_growth)
-		 && !DECL_DECLARED_INLINE_P (e->callee->decl)))
+	      && cgraph_estimate_growth (e->callee) > allowed_growth)
-	    && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+	    {
-		> e->caller->global.size + allowed_growth)
+	      if (dump_file)
-	    && cgraph_estimate_growth (e->callee) > allowed_growth)
-	  {
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
 		fprintf (dump_file,
 			 "Not inlining: code size would grow by %i.\n",
 			 cgraph_estimate_size_after_inlining (1, e->caller,
 							      e->callee)
 			 - e->caller->global.size);
-	      }
+	      continue;
-	    continue;
+	    }
-	  }
+	  if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
-	if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+					   false)
-				         false)
+	      || e->call_stmt_cannot_inline_p)
-	    || e->call_stmt_cannot_inline_p)
+	    {
-	  {
+	      if (dump_file)
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
 		fprintf (dump_file, "Not inlining: %s.\n",
 			 cgraph_inline_failed_string (e->inline_failed));
-	      }
+	      continue;
-	    continue;
+	    }
-	  }
+	  if (!e->callee->analyzed)
-	if (!e->callee->analyzed)
+	    {
-	  {
+	      if (dump_file)
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
 		fprintf (dump_file,
 			 "Not inlining: Function body no longer available.\n");
-	      }
+	      continue;
-	    continue;
+	    }
-	  }
+	  if (!tree_can_inline_p (e))
-	if (!tree_can_inline_p (e))
+	    {
-	  {
+	      if (dump_file)
-	    if (dump_file)
-	      {
-		indent_to (dump_file, depth);
 		fprintf (dump_file,
 			 "Not inlining: %s.",
-cgraph_inline_failed_string (e->inline_failed));
+			 cgraph_inline_failed_string (e->inline_failed));
-	      }
+	      continue;
-	    continue;
+	    }
-	  }
+	  if (cgraph_default_inline_p (e->callee, &failed_reason))
-	if (cgraph_default_inline_p (e->callee, &failed_reason))
+	    {
-	  inlined |= try_inline (e, mode, depth);
+	      if (dump_file)
-}
+		fprintf (dump_file, " Inlining %s into %s.\n",
-node->aux = (void *)(size_t) old_mode;
+			 cgraph_node_name (e->callee),
+			 cgraph_node_name (e->caller));
+	      cgraph_mark_inline (e);
+	      inlined = true;
+	    }
+	}
+BITMAP_FREE (visited);
+}
 return inlined;
 }
 /* Because inlining might remove no-longer reachable nodes, we need to
 keep the array visible to garbage collector to avoid reading collected
 unsigned int todo = 0;
 int iterations = 0;
 if (sorrycount || errorcount)
 return 0;
-while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
-&& cgraph_decide_inlining_incrementally (node,
+if (!optimize
-					          iterations
+|| flag_no_inline
-					          ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0))
+|| !flag_early_inlining)
 {
+/* When not optimizing or not inlining inline only always-inline
+	 functions.  */
+cgraph_decide_inlining_incrementally (node, INLINE_ALWAYS_INLINE);
 timevar_push (TV_INTEGRATION);
 todo |= optimize_inline_calls (current_function_decl);
-iterations++;
 timevar_pop (TV_INTEGRATION);
 }
-if (dump_file)
+else
-fprintf (dump_file, "Iterations: %i\n", iterations);
+{
+if (lookup_attribute ("flatten",
+			    DECL_ATTRIBUTES (node->decl)) != NULL)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "Flattening %s\n", cgraph_node_name (node));
+	  cgraph_flatten (node);
+	  timevar_push (TV_INTEGRATION);
+	  todo |= optimize_inline_calls (current_function_decl);
+	  timevar_pop (TV_INTEGRATION);
+	}
+/* We iterate incremental inlining to get trivial cases of indirect
+	 inlining.  */
+while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+	     && cgraph_decide_inlining_incrementally (node,
+						      iterations
+						      ? INLINE_SIZE_NORECURSIVE
+						      : INLINE_SIZE))
+	{
+	  timevar_push (TV_INTEGRATION);
+	  todo |= optimize_inline_calls (current_function_decl);
+	  iterations++;
+	  timevar_pop (TV_INTEGRATION);
+	}
+if (dump_file)
+	fprintf (dump_file, "Iterations: %i\n", iterations);
+}
 cfun->always_inline_functions_inlined = true;
 return todo;
-}
-/* When inlining shall be performed.  */
-static bool
-cgraph_gate_early_inlining (void)
-{
-return flag_early_inlining;
 }
 struct gimple_opt_pass pass_early_inline =
 {
 {
 GIMPLE_PASS,
 "einline",	 			/* name */
-cgraph_gate_early_inlining,		/* gate */
+NULL,					/* gate */
 cgraph_early_inlining,		/* execute */
 NULL,					/* sub */
 NULL,					/* next */
 0,					/* static_pass_number */
 TV_INLINE_HEURISTICS,			/* tv_id */
 inline_summary (node)->estimated_self_stack_size = self_stack_size;
 node->global.estimated_stack_size = self_stack_size;
 node->global.stack_frame_offset = 0;
 /* Can this function be inlined at all?  */
-node->local.inlinable = tree_inlinable_function_p (current_function_decl);
+node->local.inlinable = tree_inlinable_function_p (node->decl);
 if (node->local.inlinable && !node->local.disregard_inline_limits)
 node->local.disregard_inline_limits
-= DECL_DISREGARD_INLINE_LIMITS (current_function_decl);
+= DECL_DISREGARD_INLINE_LIMITS (node->decl);
 estimate_function_body_sizes (node);
 /* Inlining characteristics are maintained by the cgraph_mark_inline.  */
 node->global.time = inline_summary (node)->self_time;
 node->global.size = inline_summary (node)->self_size;
 return 0;
 /* This function performs intraprocedural analyzis in NODE that is required to
 inline indirect calls.  */
 static void
 inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
 {
-struct cgraph_edge *cs;
+ipa_initialize_node_params (node);
+ipa_detect_param_modifications (node);
-if (!flag_ipa_cp)
-{
-ipa_initialize_node_params (node);
-ipa_detect_param_modifications (node);
-}
 ipa_analyze_params_uses (node);
+ipa_compute_jump_functions (node);
-if (!flag_ipa_cp)
-for (cs = node->callees; cs; cs = cs->next_callee)
-{
-	ipa_count_arguments (cs);
-	ipa_compute_jump_functions (cs);
-}
 if (dump_file)
 {
 ipa_print_node_params (dump_file, node);
 ipa_print_node_jump_functions (dump_file, node);
 /* Write inline summary for node in SET.
 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
 active, we don't need to write them twice.  */
 static void
-inline_write_summary (cgraph_node_set set)
+inline_write_summary (cgraph_node_set set,
+		      varpool_node_set vset ATTRIBUTE_UNUSED)
 {
 if (flag_indirect_inlining && !flag_ipa_cp)
 ipa_prop_write_jump_functions (set);
+}
+/* When to run IPA inlining.  Inlining of always-inline functions
+happens during early inlining.  */
+static bool
+gate_cgraph_decide_inlining (void)
+{
+/* ???  We'd like to skip this if not optimizing or not inlining as
+all always-inline functions have been processed by early
+inlining already.  But this at least breaks EH with C++ as
+we need to unconditionally run fixup_cfg even at -O0.
+So leave it on unconditionally for now.  */
+return 1;
 }
 struct ipa_opt_pass_d pass_ipa_inline =
 {
 {
 IPA_PASS,
 "inline",				/* name */
-NULL,					/* gate */
+gate_cgraph_decide_inlining,		/* gate */
 cgraph_decide_inlining,		/* execute */
 NULL,					/* sub */
 NULL,					/* next */
 0,					/* static_pass_number */
 TV_INLINE_HEURISTICS,			/* tv_id */
 0,	                                /* properties_required */
 0,					/* properties_provided */
 0,					/* properties_destroyed */
 TODO_remove_functions,		/* todo_flags_finish */
 TODO_dump_cgraph | TODO_dump_func
-| TODO_remove_functions		/* todo_flags_finish */
+| TODO_remove_functions | TODO_ggc_collect	/* todo_flags_finish */
 },
 inline_generate_summary,		/* generate_summary */
 inline_write_summary,			/* write_summary */
 inline_read_summary,			/* read_summary */
-NULL,					/* function_read_summary */
+NULL,					/* write_optimization_summary */
-lto_ipa_fixup_call_notes,		/* stmt_fixup */
+NULL,					/* read_optimization_summary */
+NULL,					/* stmt_fixup */
 0,					/* TODOs */
 inline_transform,			/* function_transform */
 NULL,					/* variable_transform */
 };

Mercurial > hg > CbC > CbC_gcc

comparison gcc/ipa-inline.c @ 63:b7f97abdc517 gcc-4.6-20100522