--- a/gcc/ipa-split.c	Sun Aug 21 07:07:55 2011 +0900
+++ b/gcc/ipa-split.c	Fri Oct 27 22:46:09 2017 +0900
@@ -1,6 +1,5 @@
 /* Function splitting pass
-   Copyright (C) 2010, 2011
-   Free Software Foundation, Inc.
+   Copyright (C) 2010-2017 Free Software Foundation, Inc.
    Contributed by Jan Hubicka  <jh@suse.cz>
 
 This file is part of GCC.
@@ -78,32 +77,44 @@
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
+#include "backend.h"
+#include "rtl.h"
 #include "tree.h"
-#include "target.h"
-#include "cgraph.h"
-#include "ipa-prop.h"
-#include "tree-flow.h"
+#include "gimple.h"
+#include "cfghooks.h"
+#include "alloc-pool.h"
 #include "tree-pass.h"
-#include "flags.h"
-#include "timevar.h"
+#include "ssa.h"
+#include "cgraph.h"
 #include "diagnostic.h"
-#include "tree-dump.h"
+#include "fold-const.h"
+#include "cfganal.h"
+#include "calls.h"
+#include "gimplify.h"
+#include "gimple-iterator.h"
+#include "gimplify-me.h"
+#include "gimple-walk.h"
+#include "symbol-summary.h"
+#include "ipa-prop.h"
+#include "tree-cfg.h"
+#include "tree-into-ssa.h"
+#include "tree-dfa.h"
 #include "tree-inline.h"
-#include "fibheap.h"
 #include "params.h"
 #include "gimple-pretty-print.h"
+#include "ipa-fnsummary.h"
+#include "cfgloop.h"
+#include "tree-chkp.h"
 
 /* Per basic block info.  */
 
-typedef struct
+struct split_bb_info
 {
   unsigned int size;
   unsigned int time;
-} bb_info;
-DEF_VEC_O(bb_info);
-DEF_VEC_ALLOC_O(bb_info,heap);
+};
 
-static VEC(bb_info, heap) *bb_info_vec;
+static vec<split_bb_info> bb_info_vec;
 
 /* Description of split point.  */
 
@@ -130,13 +141,18 @@
 
 struct split_point best_split_point;
 
+/* Set of basic blocks that are not allowed to dominate a split point.  */
+
+static bitmap forbidden_dominators;
+
 static tree find_retval (basic_block return_bb);
+static tree find_retbnd (basic_block return_bb);
 
 /* Callback for walk_stmt_load_store_addr_ops.  If T is non-SSA automatic
    variable, check it if it is present in bitmap passed via DATA.  */
 
 static bool
-test_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data)
+test_nonssa_use (gimple *, tree t, tree, void *data)
 {
   t = get_base_address (t);
 
@@ -144,16 +160,21 @@
     return false;
 
   if (TREE_CODE (t) == PARM_DECL
-      || (TREE_CODE (t) == VAR_DECL
+      || (VAR_P (t)
 	  && auto_var_in_fn_p (t, current_function_decl))
       || TREE_CODE (t) == RESULT_DECL
-      || TREE_CODE (t) == LABEL_DECL)
+	 /* Normal labels are part of CFG and will be handled gratefuly.
+	    Forced labels however can be used directly by statements and
+	    need to stay in one partition along with their uses.  */
+      || (TREE_CODE (t) == LABEL_DECL
+	  && FORCED_LABEL (t)))
     return bitmap_bit_p ((bitmap)data, DECL_UID (t));
 
   /* For DECL_BY_REFERENCE, the return value is actually a pointer.  We want
      to pretend that the value pointed to is actual result decl.  */
   if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
       && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
+      && SSA_NAME_VAR (TREE_OPERAND (t, 0))
       && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
       && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
     return
@@ -169,8 +190,9 @@
 dump_split_point (FILE * file, struct split_point *current)
 {
   fprintf (file,
-	   "Split point at BB %i header time:%i header size: %i"
-	   " split time: %i split size: %i\n  bbs: ",
+	   "Split point at BB %i\n"
+	   "  header time: %i header size: %i\n"
+	   "  split time: %i split size: %i\n  bbs: ",
 	   current->entry_bb->index, current->header_time,
 	   current->header_size, current->split_time, current->split_size);
   dump_bitmap (file, current->split_bbs);
@@ -187,35 +209,35 @@
 		     basic_block return_bb)
 {
   bitmap seen = BITMAP_ALLOC (NULL);
-  VEC (basic_block,heap) *worklist = NULL;
+  vec<basic_block> worklist = vNULL;
   edge e;
   edge_iterator ei;
   bool ok = true;
+  basic_block bb;
 
   FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
-    if (e->src != ENTRY_BLOCK_PTR
+    if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
 	&& !bitmap_bit_p (current->split_bbs, e->src->index))
       {
-        VEC_safe_push (basic_block, heap, worklist, e->src);
+        worklist.safe_push (e->src);
 	bitmap_set_bit (seen, e->src->index);
       }
 
-  while (!VEC_empty (basic_block, worklist))
+  while (!worklist.is_empty ())
     {
-      gimple_stmt_iterator bsi;
-      basic_block bb = VEC_pop (basic_block, worklist);
-
+      bb = worklist.pop ();
       FOR_EACH_EDGE (e, ei, bb->preds)
-	if (e->src != ENTRY_BLOCK_PTR
+	if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
 	    && bitmap_set_bit (seen, e->src->index))
 	  {
 	    gcc_checking_assert (!bitmap_bit_p (current->split_bbs,
 					        e->src->index));
-	    VEC_safe_push (basic_block, heap, worklist, e->src);
+	    worklist.safe_push (e->src);
 	  }
-      for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+      for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
+	   gsi_next (&bsi))
 	{
-	  gimple stmt = gsi_stmt (bsi);
+	  gimple *stmt = gsi_stmt (bsi);
 	  if (is_gimple_debug (stmt))
 	    continue;
 	  if (walk_stmt_load_store_addr_ops
@@ -225,15 +247,16 @@
 	      ok = false;
 	      goto done;
 	    }
-	  if (gimple_code (stmt) == GIMPLE_LABEL
-	      && test_nonssa_use (stmt, gimple_label_label (stmt),
-				  non_ssa_vars))
-	  {
-	    ok = false;
-	    goto done;
-	  }
+	  if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
+	    if (test_nonssa_use (stmt, gimple_label_label (label_stmt),
+				 NULL_TREE, non_ssa_vars))
+	      {
+		ok = false;
+		goto done;
+	      }
 	}
-      for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+      for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
+	   gsi_next (&bsi))
 	{
 	  if (walk_stmt_load_store_addr_ops
 	      (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use,
@@ -247,16 +270,17 @@
 	{
 	  if (e->dest != return_bb)
 	    continue;
-	  for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi);
+	  for (gphi_iterator bsi = gsi_start_phis (return_bb);
+	       !gsi_end_p (bsi);
 	       gsi_next (&bsi))
 	    {
-	      gimple stmt = gsi_stmt (bsi);
+	      gphi *stmt = bsi.phi ();
 	      tree op = gimple_phi_arg_def (stmt, e->dest_idx);
 
-	      if (!is_gimple_reg (gimple_phi_result (stmt)))
+	      if (virtual_operand_p (gimple_phi_result (stmt)))
 		continue;
 	      if (TREE_CODE (op) != SSA_NAME
-		  && test_nonssa_use (stmt, op, non_ssa_vars))
+		  && test_nonssa_use (stmt, op, op, non_ssa_vars))
 		{
 		  ok = false;
 		  goto done;
@@ -264,12 +288,129 @@
 	    }
 	}
     }
+
+  /* Verify that the rest of function does not define any label
+     used by the split part.  */
+  FOR_EACH_BB_FN (bb, cfun)
+    if (!bitmap_bit_p (current->split_bbs, bb->index)
+	&& !bitmap_bit_p (seen, bb->index))
+      {
+        gimple_stmt_iterator bsi;
+        for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+	  if (glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (bsi)))
+	    {
+	      if (test_nonssa_use (label_stmt,
+				   gimple_label_label (label_stmt),
+				   NULL_TREE, non_ssa_vars))
+		{
+		  ok = false;
+		  goto done;
+		}
+	    }
+	  else
+	    break;
+      }
+    
 done:
   BITMAP_FREE (seen);
-  VEC_free (basic_block, heap, worklist);
+  worklist.release ();
   return ok;
 }
 
+/* If STMT is a call, check the callee against a list of forbidden
+   predicate functions.  If a match is found, look for uses of the
+   call result in condition statements that compare against zero.
+   For each such use, find the block targeted by the condition
+   statement for the nonzero result, and set the bit for this block
+   in the forbidden dominators bitmap.  The purpose of this is to avoid
+   selecting a split point where we are likely to lose the chance
+   to optimize away an unused function call.  */
+
+static void
+check_forbidden_calls (gimple *stmt)
+{
+  imm_use_iterator use_iter;
+  use_operand_p use_p;
+  tree lhs;
+
+  /* At the moment, __builtin_constant_p is the only forbidden
+     predicate function call (see PR49642).  */
+  if (!gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P))
+    return;
+
+  lhs = gimple_call_lhs (stmt);
+
+  if (!lhs || TREE_CODE (lhs) != SSA_NAME)
+    return;
+
+  FOR_EACH_IMM_USE_FAST (use_p, use_iter, lhs)
+    {
+      tree op1;
+      basic_block use_bb, forbidden_bb;
+      enum tree_code code;
+      edge true_edge, false_edge;
+      gcond *use_stmt;
+
+      use_stmt = dyn_cast <gcond *> (USE_STMT (use_p));
+      if (!use_stmt)
+	continue;
+
+      /* Assuming canonical form for GIMPLE_COND here, with constant
+	 in second position.  */
+      op1 = gimple_cond_rhs (use_stmt);
+      code = gimple_cond_code (use_stmt);
+      use_bb = gimple_bb (use_stmt);
+
+      extract_true_false_edges_from_block (use_bb, &true_edge, &false_edge);
+
+      /* We're only interested in comparisons that distinguish
+	 unambiguously from zero.  */
+      if (!integer_zerop (op1) || code == LE_EXPR || code == GE_EXPR)
+	continue;
+
+      if (code == EQ_EXPR)
+	forbidden_bb = false_edge->dest;
+      else
+	forbidden_bb = true_edge->dest;
+
+      bitmap_set_bit (forbidden_dominators, forbidden_bb->index);
+    }
+}
+
+/* If BB is dominated by any block in the forbidden dominators set,
+   return TRUE; else FALSE.  */
+
+static bool
+dominated_by_forbidden (basic_block bb)
+{
+  unsigned dom_bb;
+  bitmap_iterator bi;
+
+  EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators, 1, dom_bb, bi)
+    {
+      if (dominated_by_p (CDI_DOMINATORS, bb,
+			  BASIC_BLOCK_FOR_FN (cfun, dom_bb)))
+	return true;
+    }
+
+  return false;
+}
+
+/* For give split point CURRENT and return block RETURN_BB return 1
+   if ssa name VAL is set by split part and 0 otherwise.  */
+static bool
+split_part_set_ssa_name_p (tree val, struct split_point *current,
+			   basic_block return_bb)
+{
+  if (TREE_CODE (val) != SSA_NAME)
+    return false;
+
+  return (!SSA_NAME_IS_DEFAULT_DEF (val)
+	  && (bitmap_bit_p (current->split_bbs,
+			    gimple_bb (SSA_NAME_DEF_STMT (val))->index)
+	      || gimple_bb (SSA_NAME_DEF_STMT (val)) == return_bb));
+}
+
 /* We found an split_point CURRENT.  NON_SSA_VARS is bitmap of all non ssa
    variables used and RETURN_BB is return basic block.
    See if we can split function here.  */
@@ -283,27 +424,51 @@
   unsigned int call_overhead;
   edge e;
   edge_iterator ei;
-  gimple_stmt_iterator bsi;
+  gphi_iterator bsi;
   unsigned int i;
   int incoming_freq = 0;
   tree retval;
+  tree retbnd;
+  bool back_edge = false;
 
   if (dump_file && (dump_flags & TDF_DETAILS))
     dump_split_point (dump_file, current);
 
   FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
-    if (!bitmap_bit_p (current->split_bbs, e->src->index))
-      incoming_freq += EDGE_FREQUENCY (e);
+    {
+      if (e->flags & EDGE_DFS_BACK)
+	back_edge = true;
+      if (!bitmap_bit_p (current->split_bbs, e->src->index))
+        incoming_freq += EDGE_FREQUENCY (e);
+    }
 
   /* Do not split when we would end up calling function anyway.  */
   if (incoming_freq
-      >= (ENTRY_BLOCK_PTR->frequency
+      >= (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency
 	  * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100))
     {
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	fprintf (dump_file,
-		 "  Refused: incoming frequency is too large.\n");
-      return;
+      /* When profile is guessed, we can not expect it to give us
+	 realistic estimate on likelyness of function taking the
+	 complex path.  As a special case, when tail of the function is
+	 a loop, enable splitting since inlining code skipping the loop
+	 is likely noticeable win.  */
+      if (back_edge
+	  && profile_status_for_fn (cfun) != PROFILE_READ
+	  && incoming_freq < ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "  Split before loop, accepting despite low frequencies %i %i.\n",
+		     incoming_freq,
+		     ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency);
+	}
+      else
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "  Refused: incoming frequency is too large.\n");
+	  return;
+	}
     }
 
   if (!current->header_size)
@@ -317,10 +482,10 @@
      incoming from header are the same.  */
   for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi))
     {
-      gimple stmt = gsi_stmt (bsi);
+      gphi *stmt = bsi.phi ();
       tree val = NULL;
 
-      if (!is_gimple_reg (gimple_phi_result (stmt)))
+      if (virtual_operand_p (gimple_phi_result (stmt)))
 	continue;
       for (i = 0; i < gimple_phi_num_args (stmt); i++)
 	{
@@ -357,18 +522,22 @@
 	      return;
 	    }
 	}
-      else if (gimple_default_def (cfun, parm)
-	       && bitmap_bit_p (current->ssa_names_to_pass,
-				SSA_NAME_VERSION (gimple_default_def
-						  (cfun, parm))))
+      else
 	{
-	  if (!VOID_TYPE_P (TREE_TYPE (parm)))
-	    call_overhead += estimate_move_cost (TREE_TYPE (parm));
-	  num_args++;
+	  tree ddef = ssa_default_def (cfun, parm);
+	  if (ddef
+	      && bitmap_bit_p (current->ssa_names_to_pass,
+			       SSA_NAME_VERSION (ddef)))
+	    {
+	      if (!VOID_TYPE_P (TREE_TYPE (parm)))
+		call_overhead += estimate_move_cost (TREE_TYPE (parm), false);
+	      num_args++;
+	    }
 	}
     }
   if (!VOID_TYPE_P (TREE_TYPE (current_function_decl)))
-    call_overhead += estimate_move_cost (TREE_TYPE (current_function_decl));
+    call_overhead += estimate_move_cost (TREE_TYPE (current_function_decl),
+					 false);
 
   if (current->split_size <= call_overhead)
     {
@@ -388,6 +557,31 @@
       return;
     }
 
+  /* Splitting functions brings the target out of comdat group; this will
+     lead to code duplication if the function is reused by other unit.
+     Limit this duplication.  This is consistent with limit in tree-sra.c  
+     FIXME: with LTO we ought to be able to do better!  */
+  if (DECL_ONE_ONLY (current_function_decl)
+      && current->split_size >= (unsigned int) MAX_INLINE_INSNS_AUTO)
+    {
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	fprintf (dump_file,
+		 "  Refused: function is COMDAT and tail is too large\n");
+      return;
+    }
+  /* For comdat functions also reject very small tails; those will likely get
+     inlined back and we do not want to risk the duplication overhead.
+     FIXME: with LTO we ought to be able to do better!  */
+  if (DECL_ONE_ONLY (current_function_decl)
+      && current->split_size
+	 <= (unsigned int) PARAM_VALUE (PARAM_EARLY_INLINING_INSNS) / 2)
+    {
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	fprintf (dump_file,
+		 "  Refused: function is COMDAT and tail is too small\n");
+      return;
+    }
+
   /* FIXME: we currently can pass only SSA function parameters to the split
      arguments.  Once parm_adjustment infrastructure is supported by cloning,
      we can pass more than that.  */
@@ -411,6 +605,18 @@
 		 "  Refused: split part has non-ssa uses\n");
       return;
     }
+
+  /* If the split point is dominated by a forbidden block, reject
+     the split.  */
+  if (!bitmap_empty_p (forbidden_dominators)
+      && dominated_by_forbidden (current->entry_bb))
+    {
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	fprintf (dump_file,
+		 "  Refused: split point dominated by forbidden block\n");
+      return;
+    }
+
   /* See if retval used by return bb is computed by header or split part.
      When it is computed by split part, we need to produce return statement
      in the split part and add code to header to pass it around.
@@ -423,44 +629,76 @@
        4) For non-SSA we need to look where the var is computed. */
   retval = find_retval (return_bb);
   if (!retval)
-    current->split_part_set_retval = true;
+    {
+      /* If there is a return_bb with no return value in function returning
+	 value by reference, also make the split part return void, otherwise
+	 we expansion would try to create a non-POD temporary, which is
+	 invalid.  */
+      if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
+	  && DECL_RESULT (current_function_decl)
+	  && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+	current->split_part_set_retval = false;
+      else
+	current->split_part_set_retval = true;
+    }
   else if (is_gimple_min_invariant (retval))
     current->split_part_set_retval = false;
   /* Special case is value returned by reference we record as if it was non-ssa
      set to result_decl.  */
   else if (TREE_CODE (retval) == SSA_NAME
+	   && SSA_NAME_VAR (retval)
 	   && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL
 	   && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
     current->split_part_set_retval
        = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval)));
   else if (TREE_CODE (retval) == SSA_NAME)
     current->split_part_set_retval
-      = (!SSA_NAME_IS_DEFAULT_DEF (retval)
-	 && (bitmap_bit_p (current->split_bbs,
-			  gimple_bb (SSA_NAME_DEF_STMT (retval))->index)
-	     || gimple_bb (SSA_NAME_DEF_STMT (retval)) == return_bb));
+      = split_part_set_ssa_name_p (retval, current, return_bb);
   else if (TREE_CODE (retval) == PARM_DECL)
     current->split_part_set_retval = false;
-  else if (TREE_CODE (retval) == VAR_DECL
+  else if (VAR_P (retval)
 	   || TREE_CODE (retval) == RESULT_DECL)
     current->split_part_set_retval
       = bitmap_bit_p (non_ssa_vars, DECL_UID (retval));
   else
     current->split_part_set_retval = true;
 
+  /* See if retbnd used by return bb is computed by header or split part.  */
+  retbnd = find_retbnd (return_bb);
+  if (retbnd)
+    {
+      bool split_part_set_retbnd
+	= split_part_set_ssa_name_p (retbnd, current, return_bb);
+
+      /* If we have both return value and bounds then keep their definitions
+	 in a single function.  We use SSA names to link returned bounds and
+	 value and therefore do not handle cases when result is passed by
+	 reference (which should not be our case anyway since bounds are
+	 returned for pointers only).  */
+      if ((DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
+	   && current->split_part_set_retval)
+	  || split_part_set_retbnd != current->split_part_set_retval)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "  Refused: split point splits return value and bounds\n");
+	  return;
+	}
+    }
+
   /* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
      for the return value.  If there are other PHIs, give up.  */
-  if (return_bb != EXIT_BLOCK_PTR)
+  if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
     {
-      gimple_stmt_iterator psi;
+      gphi_iterator psi;
 
       for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi))
-	if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi)))
+	if (!virtual_operand_p (gimple_phi_result (psi.phi ()))
 	    && !(retval
 		 && current->split_part_set_retval
 		 && TREE_CODE (retval) == SSA_NAME
 		 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
-		 && SSA_NAME_DEF_STMT (retval) == gsi_stmt (psi)))
+		 && SSA_NAME_DEF_STMT (retval) == psi.phi ()))
 	  {
 	    if (dump_file && (dump_flags & TDF_DETAILS))
 	      fprintf (dump_file,
@@ -501,8 +739,9 @@
    of the form:
    <retval> = tmp_var;
    return <retval>
-   but return_bb can not be more complex than this.
-   If nothing is found, return EXIT_BLOCK_PTR.
+   but return_bb can not be more complex than this (except for
+   -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there).
+   If nothing is found, return the exit block.
 
    When there are multiple RETURN statement, chose one with return value,
    since that one is more likely shared by multiple code paths.
@@ -517,19 +756,21 @@
 find_return_bb (void)
 {
   edge e;
-  basic_block return_bb = EXIT_BLOCK_PTR;
+  basic_block return_bb = EXIT_BLOCK_PTR_FOR_FN (cfun);
   gimple_stmt_iterator bsi;
   bool found_return = false;
   tree retval = NULL_TREE;
 
-  if (!single_pred_p (EXIT_BLOCK_PTR))
+  if (!single_pred_p (EXIT_BLOCK_PTR_FOR_FN (cfun)))
     return return_bb;
 
-  e = single_pred_edge (EXIT_BLOCK_PTR);
+  e = single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun));
   for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi))
     {
-      gimple stmt = gsi_stmt (bsi);
-      if (gimple_code (stmt) == GIMPLE_LABEL || is_gimple_debug (stmt))
+      gimple *stmt = gsi_stmt (bsi);
+      if (gimple_code (stmt) == GIMPLE_LABEL
+	  || is_gimple_debug (stmt)
+	  || gimple_clobber_p (stmt))
 	;
       else if (gimple_code (stmt) == GIMPLE_ASSIGN
 	       && found_return
@@ -539,11 +780,16 @@
 		   || is_gimple_min_invariant (gimple_assign_rhs1 (stmt)))
 	       && retval == gimple_assign_lhs (stmt))
 	;
-      else if (gimple_code (stmt) == GIMPLE_RETURN)
+      else if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
 	{
 	  found_return = true;
-	  retval = gimple_return_retval (stmt);
+	  retval = gimple_return_retval (return_stmt);
 	}
+      /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return
+	 bb.  */
+      else if ((flag_sanitize & SANITIZE_THREAD)
+	       && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
+	;
       else
 	break;
     }
@@ -560,10 +806,23 @@
 {
   gimple_stmt_iterator bsi;
   for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
+    if (greturn *return_stmt = dyn_cast <greturn *> (gsi_stmt (bsi)))
+      return gimple_return_retval (return_stmt);
+    else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
+	     && !gimple_clobber_p (gsi_stmt (bsi)))
+      return gimple_assign_rhs1 (gsi_stmt (bsi));
+  return NULL;
+}
+
+/* Given return basic block RETURN_BB, see where return bounds are really
+   stored.  */
+static tree
+find_retbnd (basic_block return_bb)
+{
+  gimple_stmt_iterator bsi;
+  for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi); gsi_prev (&bsi))
     if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
-      return gimple_return_retval (gsi_stmt (bsi));
-    else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN)
-      return gimple_assign_rhs1 (gsi_stmt (bsi));
+      return gimple_return_retbnd (gsi_stmt (bsi));
   return NULL;
 }
 
@@ -572,7 +831,7 @@
    Return true when access to T prevents splitting the function.  */
 
 static bool
-mark_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data)
+mark_nonssa_use (gimple *, tree t, tree, void *data)
 {
   t = get_base_address (t);
 
@@ -589,16 +848,16 @@
       return true;
     }
 
-  if ((TREE_CODE (t) == VAR_DECL
-       && auto_var_in_fn_p (t, current_function_decl))
+  if ((VAR_P (t) && auto_var_in_fn_p (t, current_function_decl))
       || TREE_CODE (t) == RESULT_DECL
-      || TREE_CODE (t) == LABEL_DECL)
+      || (TREE_CODE (t) == LABEL_DECL && FORCED_LABEL (t)))
     bitmap_set_bit ((bitmap)data, DECL_UID (t));
 
   /* For DECL_BY_REFERENCE, the return value is actually a pointer.  We want
      to pretend that the value pointed to is actual result decl.  */
   if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
       && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
+      && SSA_NAME_VAR (TREE_OPERAND (t, 0))
       && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
       && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
     return
@@ -623,14 +882,14 @@
 	  bitmap set_ssa_names, bitmap used_ssa_names,
 	  bitmap non_ssa_vars)
 {
-  gimple_stmt_iterator bsi;
   edge e;
   edge_iterator ei;
   bool can_split = true;
 
-  for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+  for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
+       gsi_next (&bsi))
     {
-      gimple stmt = gsi_stmt (bsi);
+      gimple *stmt = gsi_stmt (bsi);
       tree op;
       ssa_op_iter iter;
       tree decl;
@@ -638,6 +897,9 @@
       if (is_gimple_debug (stmt))
 	continue;
 
+      if (gimple_clobber_p (stmt))
+	continue;
+
       /* FIXME: We can split regions containing EH.  We can not however
 	 split RESX, EH_DISPATCH and EH_POINTER referring to same region
 	 into different partitions.  This would require tracking of
@@ -694,14 +956,13 @@
 						   mark_nonssa_use,
 						   mark_nonssa_use);
     }
-  for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+  for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
+       gsi_next (&bsi))
     {
-      gimple stmt = gsi_stmt (bsi);
+      gphi *stmt = bsi.phi ();
       unsigned int i;
 
-      if (is_gimple_debug (stmt))
-	continue;
-      if (!is_gimple_reg (gimple_phi_result (stmt)))
+      if (virtual_operand_p (gimple_phi_result (stmt)))
 	continue;
       bitmap_set_bit (set_ssa_names,
 		      SSA_NAME_VERSION (gimple_phi_result (stmt)));
@@ -720,19 +981,19 @@
   FOR_EACH_EDGE (e, ei, bb->succs)
     if (e->dest == return_bb)
       {
-	for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
+	for (gphi_iterator bsi = gsi_start_phis (return_bb);
+	     !gsi_end_p (bsi);
+	     gsi_next (&bsi))
 	  {
-	    gimple stmt = gsi_stmt (bsi);
+	    gphi *stmt = bsi.phi ();
 	    tree op = gimple_phi_arg_def (stmt, e->dest_idx);
 
-	    if (is_gimple_debug (stmt))
-	      continue;
-	    if (!is_gimple_reg (gimple_phi_result (stmt)))
+	    if (virtual_operand_p (gimple_phi_result (stmt)))
 	      continue;
 	    if (TREE_CODE (op) == SSA_NAME)
 	      bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
 	    else
-	      can_split &= !mark_nonssa_use (stmt, op, non_ssa_vars);
+	      can_split &= !mark_nonssa_use (stmt, op, op, non_ssa_vars);
 	  }
       }
   return can_split;
@@ -740,7 +1001,7 @@
 
 /* Stack entry for recursive DFS walk in find_split_point.  */
 
-typedef struct
+struct stack_entry
 {
   /* Basic block we are examining.  */
   basic_block bb;
@@ -766,9 +1027,7 @@
 
   /* When false we can not split on this BB.  */
   bool can_split;
-} stack_entry;
-DEF_VEC_O(stack_entry);
-DEF_VEC_ALLOC_O(stack_entry,heap);
+};
 
 
 /* Find all articulations and call consider_split on them.
@@ -789,12 +1048,11 @@
    the component used by consider_split.  */
 
 static void
-find_split_points (int overall_time, int overall_size)
+find_split_points (basic_block return_bb, int overall_time, int overall_size)
 {
   stack_entry first;
-  VEC(stack_entry, heap) *stack = NULL;
+  vec<stack_entry> stack = vNULL;
   basic_block bb;
-  basic_block return_bb = find_return_bb ();
   struct split_point current;
 
   current.header_time = overall_time;
@@ -803,29 +1061,31 @@
   current.split_size = 0;
   current.ssa_names_to_pass = BITMAP_ALLOC (NULL);
 
-  first.bb = ENTRY_BLOCK_PTR;
+  first.bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
   first.edge_num = 0;
   first.overall_time = 0;
   first.overall_size = 0;
   first.earliest = INT_MAX;
   first.set_ssa_names = 0;
   first.used_ssa_names = 0;
+  first.non_ssa_vars = 0;
   first.bbs_visited = 0;
-  VEC_safe_push (stack_entry, heap, stack, &first);
-  ENTRY_BLOCK_PTR->aux = (void *)(intptr_t)-1;
+  first.can_split = false;
+  stack.safe_push (first);
+  ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(intptr_t)-1;
 
-  while (!VEC_empty (stack_entry, stack))
+  while (!stack.is_empty ())
     {
-      stack_entry *entry = VEC_last (stack_entry, stack);
+      stack_entry *entry = &stack.last ();
 
       /* We are walking an acyclic graph, so edge_num counts
 	 succ and pred edges together.  However when considering
          articulation, we want to have processed everything reachable
 	 from articulation but nothing that reaches into it.  */
       if (entry->edge_num == EDGE_COUNT (entry->bb->succs)
-	  && entry->bb != ENTRY_BLOCK_PTR)
+	  && entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
 	{
-	  int pos = VEC_length (stack_entry, stack);
+	  int pos = stack.length ();
 	  entry->can_split &= visit_bb (entry->bb, return_bb,
 					entry->set_ssa_names,
 					entry->used_ssa_names,
@@ -875,7 +1135,7 @@
 	  entry->edge_num++;
 
 	  /* New BB to visit, push it to the stack.  */
-	  if (dest != return_bb && dest != EXIT_BLOCK_PTR
+	  if (dest != return_bb && dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
 	      && !dest->aux)
 	    {
 	      stack_entry new_entry;
@@ -883,9 +1143,9 @@
 	      new_entry.bb = dest;
 	      new_entry.edge_num = 0;
 	      new_entry.overall_time
-		 = VEC_index (bb_info, bb_info_vec, dest->index)->time;
+		 = bb_info_vec[dest->index].time;
 	      new_entry.overall_size
-		 = VEC_index (bb_info, bb_info_vec, dest->index)->size;
+		 = bb_info_vec[dest->index].size;
 	      new_entry.earliest = INT_MAX;
 	      new_entry.set_ssa_names = BITMAP_ALLOC (NULL);
 	      new_entry.used_ssa_names = BITMAP_ALLOC (NULL);
@@ -893,8 +1153,8 @@
 	      new_entry.non_ssa_vars = BITMAP_ALLOC (NULL);
 	      new_entry.can_split = true;
 	      bitmap_set_bit (new_entry.bbs_visited, dest->index);
-	      VEC_safe_push (stack_entry, heap, stack, &new_entry);
-	      dest->aux = (void *)(intptr_t)VEC_length (stack_entry, stack);
+	      stack.safe_push (new_entry);
+	      dest->aux = (void *)(intptr_t)stack.length ();
 	    }
 	  /* Back edge found, record the earliest point.  */
 	  else if ((intptr_t)dest->aux > 0
@@ -903,10 +1163,9 @@
 	}
       /* We are done with examining the edges.  Pop off the value from stack
 	 and merge stuff we accumulate during the walk.  */
-      else if (entry->bb != ENTRY_BLOCK_PTR)
+      else if (entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
 	{
-	  stack_entry *prev = VEC_index (stack_entry, stack,
-					 VEC_length (stack_entry, stack) - 2);
+	  stack_entry *prev = &stack[stack.length () - 2];
 
 	  entry->bb->aux = (void *)(intptr_t)-1;
 	  prev->can_split &= entry->can_split;
@@ -925,40 +1184,38 @@
 	  BITMAP_FREE (entry->used_ssa_names);
 	  BITMAP_FREE (entry->bbs_visited);
 	  BITMAP_FREE (entry->non_ssa_vars);
-	  VEC_pop (stack_entry, stack);
+	  stack.pop ();
 	}
       else
-        VEC_pop (stack_entry, stack);
+        stack.pop ();
     }
-  ENTRY_BLOCK_PTR->aux = NULL;
-  FOR_EACH_BB (bb)
+  ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = NULL;
+  FOR_EACH_BB_FN (bb, cfun)
     bb->aux = NULL;
-  VEC_free (stack_entry, heap, stack);
+  stack.release ();
   BITMAP_FREE (current.ssa_names_to_pass);
 }
 
 /* Split function at SPLIT_POINT.  */
 
 static void
-split_function (struct split_point *split_point)
+split_function (basic_block return_bb, struct split_point *split_point,
+		bool add_tsan_func_exit)
 {
-  VEC (tree, heap) *args_to_pass = NULL;
-  bitmap args_to_skip = BITMAP_ALLOC (NULL);
+  vec<tree> args_to_pass = vNULL;
+  bitmap args_to_skip;
   tree parm;
   int num = 0;
-  struct cgraph_node *node;
-  basic_block return_bb = find_return_bb ();
+  cgraph_node *node, *cur_node = cgraph_node::get (current_function_decl);
   basic_block call_bb;
-  gimple_stmt_iterator gsi;
-  gimple call;
+  gcall *call, *tsan_func_exit_call = NULL;
   edge e;
   edge_iterator ei;
-  tree retval = NULL, real_retval = NULL;
-  bool split_part_return_p = false;
-  gimple last_stmt = NULL;
-  bool conv_needed = false;
+  tree retval = NULL, real_retval = NULL, retbnd = NULL;
+  bool with_bounds = chkp_function_instrumented_p (current_function_decl);
+  gimple *last_stmt = NULL;
   unsigned int i;
-  tree arg;
+  tree arg, ddef;
 
   if (dump_file)
     {
@@ -966,102 +1223,153 @@
       dump_split_point (dump_file, split_point);
     }
 
+  if (cur_node->local.can_change_signature)
+    args_to_skip = BITMAP_ALLOC (NULL);
+  else
+    args_to_skip = NULL;
+
   /* Collect the parameters of new function and args_to_skip bitmap.  */
   for (parm = DECL_ARGUMENTS (current_function_decl);
        parm; parm = DECL_CHAIN (parm), num++)
-    if (!is_gimple_reg (parm)
-	|| !gimple_default_def (cfun, parm)
-	|| !bitmap_bit_p (split_point->ssa_names_to_pass,
-			  SSA_NAME_VERSION (gimple_default_def (cfun, parm))))
+    if (args_to_skip
+	&& (!is_gimple_reg (parm)
+	    || (ddef = ssa_default_def (cfun, parm)) == NULL_TREE
+	    || !bitmap_bit_p (split_point->ssa_names_to_pass,
+			      SSA_NAME_VERSION (ddef))))
       bitmap_set_bit (args_to_skip, num);
     else
       {
-	arg = gimple_default_def (cfun, parm);
-	if (TYPE_MAIN_VARIANT (DECL_ARG_TYPE (parm))
-	    != TYPE_MAIN_VARIANT (TREE_TYPE (arg)))
-	  {
-	    conv_needed = true;
-	    arg = fold_convert (DECL_ARG_TYPE (parm), arg);
-	  }
-	VEC_safe_push (tree, heap, args_to_pass, arg);
+	/* This parm might not have been used up to now, but is going to be
+	   used, hence register it.  */
+	if (is_gimple_reg (parm))
+	  arg = get_or_create_ssa_default_def (cfun, parm);
+	else
+	  arg = parm;
+
+	if (!useless_type_conversion_p (DECL_ARG_TYPE (parm), TREE_TYPE (arg)))
+	  arg = fold_convert (DECL_ARG_TYPE (parm), arg);
+	args_to_pass.safe_push (arg);
       }
 
   /* See if the split function will return.  */
+  bool split_part_return_p = false;
   FOR_EACH_EDGE (e, ei, return_bb->preds)
-    if (bitmap_bit_p (split_point->split_bbs, e->src->index))
-      break;
-  if (e)
-    split_part_return_p = true;
+    {
+      if (bitmap_bit_p (split_point->split_bbs, e->src->index))
+	split_part_return_p = true;
+    }
 
   /* Add return block to what will become the split function.
      We do not return; no return block is needed.  */
   if (!split_part_return_p)
     ;
   /* We have no return block, so nothing is needed.  */
-  else if (return_bb == EXIT_BLOCK_PTR)
+  else if (return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
     ;
   /* When we do not want to return value, we need to construct
      new return block with empty return statement.
      FIXME: Once we are able to change return type, we should change function
      to return void instead of just outputting function with undefined return
      value.  For structures this affects quality of codegen.  */
-  else if (!split_point->split_part_set_retval
-           && find_retval (return_bb))
+  else if ((retval = find_retval (return_bb))
+	   && !split_point->split_part_set_retval)
     {
       bool redirected = true;
       basic_block new_return_bb = create_basic_block (NULL, 0, return_bb);
       gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb);
       gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT);
+      new_return_bb->count = profile_count::zero ();
       while (redirected)
 	{
 	  redirected = false;
 	  FOR_EACH_EDGE (e, ei, return_bb->preds)
 	    if (bitmap_bit_p (split_point->split_bbs, e->src->index))
 	      {
-		new_return_bb->count += e->count;
 		new_return_bb->frequency += EDGE_FREQUENCY (e);
 		redirect_edge_and_branch (e, new_return_bb);
 		redirected = true;
 		break;
 	      }
 	}
-      e = make_edge (new_return_bb, EXIT_BLOCK_PTR, 0);
-      e->probability = REG_BR_PROB_BASE;
-      e->count = new_return_bb->count;
+      e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
+      add_bb_to_loop (new_return_bb, current_loops->tree_root);
       bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
+      retbnd = find_retbnd (return_bb);
     }
   /* When we pass around the value, use existing return block.  */
   else
-    bitmap_set_bit (split_point->split_bbs, return_bb->index);
+    {
+      bitmap_set_bit (split_point->split_bbs, return_bb->index);
+      retbnd = find_retbnd (return_bb);
+    }
 
   /* If RETURN_BB has virtual operand PHIs, they must be removed and the
      virtual operand marked for renaming as we change the CFG in a way that
-     tree-inline is not able to compensate for. 
+     tree-inline is not able to compensate for.
 
      Note this can happen whether or not we have a return value.  If we have
      a return value, then RETURN_BB may have PHIs for real operands too.  */
-  if (return_bb != EXIT_BLOCK_PTR)
+  if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
     {
-      for (gsi = gsi_start_phis (return_bb); !gsi_end_p (gsi);)
+      bool phi_p = false;
+      for (gphi_iterator gsi = gsi_start_phis (return_bb);
+	   !gsi_end_p (gsi);)
 	{
-	  gimple stmt = gsi_stmt (gsi);
-	  if (is_gimple_reg (gimple_phi_result (stmt)))
+	  gphi *stmt = gsi.phi ();
+	  if (!virtual_operand_p (gimple_phi_result (stmt)))
 	    {
 	      gsi_next (&gsi);
 	      continue;
 	    }
 	  mark_virtual_phi_result_for_renaming (stmt);
 	  remove_phi_node (&gsi, true);
+	  phi_p = true;
 	}
+      /* In reality we have to rename the reaching definition of the
+	 virtual operand at return_bb as we will eventually release it
+	 when we remove the code region we outlined.
+	 So we have to rename all immediate virtual uses of that region
+	 if we didn't see a PHI definition yet.  */
+      /* ???  In real reality we want to set the reaching vdef of the
+         entry of the SESE region as the vuse of the call and the reaching
+	 vdef of the exit of the SESE region as the vdef of the call.  */
+      if (!phi_p)
+	for (gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
+	     !gsi_end_p (gsi);
+	     gsi_next (&gsi))
+	  {
+	    gimple *stmt = gsi_stmt (gsi);
+	    if (gimple_vuse (stmt))
+	      {
+		gimple_set_vuse (stmt, NULL_TREE);
+		update_stmt (stmt);
+	      }
+	    if (gimple_vdef (stmt))
+	      break;
+	  }
     }
 
   /* Now create the actual clone.  */
-  rebuild_cgraph_edges ();
-  node = cgraph_function_versioning (cgraph_node (current_function_decl),
-				     NULL, NULL,
-				     args_to_skip,
-				     split_point->split_bbs,
-				     split_point->entry_bb, "part");
+  cgraph_edge::rebuild_edges ();
+  node = cur_node->create_version_clone_with_body
+    (vNULL, NULL, args_to_skip,
+     !split_part_return_p || !split_point->split_part_set_retval,
+     split_point->split_bbs, split_point->entry_bb, "part");
+
+  node->split_part = true;
+
+  if (cur_node->same_comdat_group)
+    {
+      /* TODO: call is versionable if we make sure that all
+	 callers are inside of a comdat group.  */
+      cur_node->calls_comdat_local = 1;
+      node->add_to_same_comdat_group (cur_node);
+    }
+
+
+  /* Let's take a time profile for splitted function.  */
+  node->tp_first_run = cur_node->tp_first_run + 1;
+
   /* For usual cloning it is enough to clear builtin only when signature
      changes.  For partial inlining we however can not expect the part
      of builtin implementation to have same semantic as the whole.  */
@@ -1070,7 +1378,55 @@
       DECL_BUILT_IN_CLASS (node->decl) = NOT_BUILT_IN;
       DECL_FUNCTION_CODE (node->decl) = (enum built_in_function) 0;
     }
-  cgraph_node_remove_callees (cgraph_node (current_function_decl));
+
+  /* If return_bb contains any clobbers that refer to SSA_NAMEs
+     set in the split part, remove them.  Also reset debug stmts that
+     refer to SSA_NAMEs set in the split part.  */
+  if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
+    {
+      gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
+      while (!gsi_end_p (gsi))
+	{
+	  tree op;
+	  ssa_op_iter iter;
+	  gimple *stmt = gsi_stmt (gsi);
+	  bool remove = false;
+	  if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
+	    FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
+	      {
+		basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
+		if (op != retval
+		    && bb
+		    && bb != return_bb
+		    && bitmap_bit_p (split_point->split_bbs, bb->index))
+		  {
+		    if (is_gimple_debug (stmt))
+		      {
+			gimple_debug_bind_reset_value (stmt);
+			update_stmt (stmt);
+		      }
+		    else
+		      remove = true;
+		    break;
+		  }
+	      }
+	  if (remove)
+	    gsi_remove (&gsi, true);
+	  else
+	    gsi_next (&gsi);
+	}
+    }
+
+  /* If the original function is instrumented then it's
+     part is also instrumented.  */
+  if (with_bounds)
+    chkp_function_mark_instrumented (node->decl);
+
+  /* If the original function is declared inline, there is no point in issuing
+     a warning for the non-inlinable part.  */
+  DECL_NO_INLINE_WARNING_P (node->decl) = 1;
+  cur_node->remove_callees ();
+  cur_node->remove_all_references ();
   if (!split_part_return_p)
     TREE_THIS_VOLATILE (node->decl) = 1;
   if (dump_file)
@@ -1079,7 +1435,7 @@
   /* Create the basic block we place call into.  It is the entry basic block
      split after last label.  */
   call_bb = split_point->entry_bb;
-  for (gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);)
+  for (gimple_stmt_iterator gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);)
     if (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL)
       {
 	last_stmt = gsi_stmt (gsi);
@@ -1091,46 +1447,95 @@
   remove_edge (e);
 
   /* Produce the call statement.  */
-  gsi = gsi_last_bb (call_bb);
-  if (conv_needed)
-    FOR_EACH_VEC_ELT (tree, args_to_pass, i, arg)
-      if (!is_gimple_val (arg))
+  gimple_stmt_iterator gsi = gsi_last_bb (call_bb);
+  FOR_EACH_VEC_ELT (args_to_pass, i, arg)
+    if (!is_gimple_val (arg))
+      {
+	arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE,
+					false, GSI_CONTINUE_LINKING);
+	args_to_pass[i] = arg;
+      }
+  call = gimple_build_call_vec (node->decl, args_to_pass);
+  gimple_call_set_with_bounds (call, with_bounds);
+  gimple_set_block (call, DECL_INITIAL (current_function_decl));
+  args_to_pass.release ();
+
+  /* For optimized away parameters, add on the caller side
+     before the call
+     DEBUG D#X => parm_Y(D)
+     stmts and associate D#X with parm in decl_debug_args_lookup
+     vector to say for debug info that if parameter parm had been passed,
+     it would have value parm_Y(D).  */
+  if (args_to_skip)
+    {
+      vec<tree, va_gc> **debug_args = NULL;
+      unsigned i = 0, len = 0;
+      if (MAY_HAVE_DEBUG_STMTS)
 	{
-	  arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE,
-					  false, GSI_NEW_STMT);
-	  VEC_replace (tree, args_to_pass, i, arg);
+	  debug_args = decl_debug_args_lookup (node->decl);
+	  if (debug_args)
+	    len = vec_safe_length (*debug_args);
 	}
-  call = gimple_build_call_vec (node->decl, args_to_pass);
-  gimple_set_block (call, DECL_INITIAL (current_function_decl));
+      for (parm = DECL_ARGUMENTS (current_function_decl), num = 0;
+	   parm; parm = DECL_CHAIN (parm), num++)
+	if (bitmap_bit_p (args_to_skip, num) && is_gimple_reg (parm))
+	  {
+	    tree ddecl;
+	    gimple *def_temp;
+
+	    /* This needs to be done even without MAY_HAVE_DEBUG_STMTS,
+	       otherwise if it didn't exist before, we'd end up with
+	       different SSA_NAME_VERSIONs between -g and -g0.  */
+	    arg = get_or_create_ssa_default_def (cfun, parm);
+	    if (!MAY_HAVE_DEBUG_STMTS || debug_args == NULL)
+	      continue;
+
+	    while (i < len && (**debug_args)[i] != DECL_ORIGIN (parm))
+	      i += 2;
+	    if (i >= len)
+	      continue;
+	    ddecl = (**debug_args)[i + 1];
+	    def_temp
+	      = gimple_build_debug_bind (ddecl, unshare_expr (arg), call);
+	    gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
+	  }
+    }
 
   /* We avoid address being taken on any variable used by split part,
      so return slot optimization is always possible.  Moreover this is
      required to make DECL_BY_REFERENCE work.  */
   if (aggregate_value_p (DECL_RESULT (current_function_decl),
-			 TREE_TYPE (current_function_decl)))
+			 TREE_TYPE (current_function_decl))
+      && (!is_gimple_reg_type (TREE_TYPE (DECL_RESULT (current_function_decl)))
+	  || DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))))
     gimple_call_set_return_slot_opt (call, true);
 
+  if (add_tsan_func_exit)
+    tsan_func_exit_call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0);
+
   /* Update return value.  This is bit tricky.  When we do not return,
      do nothing.  When we return we might need to update return_bb
      or produce a new return statement.  */
   if (!split_part_return_p)
-    gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+    {
+      gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+      if (tsan_func_exit_call)
+	gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
+    }
   else
     {
-      e = make_edge (call_bb, return_bb,
-		     return_bb == EXIT_BLOCK_PTR ? 0 : EDGE_FALLTHRU);
-      e->count = call_bb->count;
-      e->probability = REG_BR_PROB_BASE;
+      e = make_single_succ_edge (call_bb, return_bb,
+				 return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
+				 ? 0 : EDGE_FALLTHRU);
 
       /* If there is return basic block, see what value we need to store
          return value into and put call just before it.  */
-      if (return_bb != EXIT_BLOCK_PTR)
+      if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
 	{
-	  real_retval = retval = find_retval (return_bb);
-
+	  real_retval = retval;
 	  if (real_retval && split_point->split_part_set_retval)
 	    {
-	      gimple_stmt_iterator psi;
+	      gphi_iterator psi;
 
 	      /* See if we need new SSA_NAME for the result.
 		 When DECL_BY_REFERENCE is true, retval is actually pointer to
@@ -1138,18 +1543,18 @@
 	      if (TREE_CODE (retval) == SSA_NAME
 		  && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
 		{
-		  retval = make_ssa_name (SSA_NAME_VAR (retval), call);
+		  retval = copy_ssa_name (retval, call);
 
 		  /* See if there is PHI defining return value.  */
 		  for (psi = gsi_start_phis (return_bb);
 		       !gsi_end_p (psi); gsi_next (&psi))
-		    if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi))))
+		    if (!virtual_operand_p (gimple_phi_result (psi.phi ())))
 		      break;
 
 		  /* When there is PHI, just update its value.  */
 		  if (TREE_CODE (retval) == SSA_NAME
 		      && !gsi_end_p (psi))
-		    add_phi_arg (gsi_stmt (psi), retval, e, UNKNOWN_LOCATION);
+		    add_phi_arg (psi.phi (), retval, e, UNKNOWN_LOCATION);
 		  /* Otherwise update the return BB itself.
 		     find_return_bb allows at most one assignment to return value,
 		     so update first statement.  */
@@ -1158,62 +1563,117 @@
 		      gimple_stmt_iterator bsi;
 		      for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
 			   gsi_next (&bsi))
-			if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
+			if (greturn *return_stmt
+			      = dyn_cast <greturn *> (gsi_stmt (bsi)))
 			  {
-			    gimple_return_set_retval (gsi_stmt (bsi), retval);
+			    gimple_return_set_retval (return_stmt, retval);
 			    break;
 			  }
-			else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN)
+			else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
+				 && !gimple_clobber_p (gsi_stmt (bsi)))
 			  {
 			    gimple_assign_set_rhs1 (gsi_stmt (bsi), retval);
 			    break;
 			  }
 		      update_stmt (gsi_stmt (bsi));
+		      /* Also adjust clobbers and debug stmts in return_bb.  */
+		      for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
+			   gsi_next (&bsi))
+			{
+			  gimple *stmt = gsi_stmt (bsi);
+			  if (gimple_clobber_p (stmt)
+			      || is_gimple_debug (stmt))
+			    {
+			      ssa_op_iter iter;
+			      use_operand_p use_p;
+			      bool update = false;
+			      FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter,
+							SSA_OP_USE)
+				if (USE_FROM_PTR (use_p) == real_retval)
+				  {
+				    SET_USE (use_p, retval);
+				    update = true;
+				  }
+			      if (update)
+				update_stmt (stmt);
+			    }
+			}
+		    }
+
+		  /* Replace retbnd with new one.  */
+		  if (retbnd)
+		    {
+		      gimple_stmt_iterator bsi;
+		      for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi);
+			   gsi_prev (&bsi))
+			if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
+			  {
+			    retbnd = copy_ssa_name (retbnd, call);
+			    gimple_return_set_retbnd (gsi_stmt (bsi), retbnd);
+			    update_stmt (gsi_stmt (bsi));
+			    break;
+			  }
 		    }
 		}
 	      if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
-	        gimple_call_set_lhs (call, build_simple_mem_ref (retval));
+		{
+		  gimple_call_set_lhs (call, build_simple_mem_ref (retval));
+		  gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+		}
 	      else
-	        gimple_call_set_lhs (call, retval);
+		{
+		  tree restype;
+		  restype = TREE_TYPE (DECL_RESULT (current_function_decl));
+		  gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+		  if (!useless_type_conversion_p (TREE_TYPE (retval), restype))
+		    {
+		      gimple *cpy;
+		      tree tem = create_tmp_reg (restype);
+		      tem = make_ssa_name (tem, call);
+		      cpy = gimple_build_assign (retval, NOP_EXPR, tem);
+		      gsi_insert_after (&gsi, cpy, GSI_NEW_STMT);
+		      retval = tem;
+		    }
+		  /* Build bndret call to obtain returned bounds.  */
+		  if (retbnd)
+		    chkp_insert_retbnd_call (retbnd, retval, &gsi);
+		  gimple_call_set_lhs (call, retval);
+		  update_stmt (call);
+		}
 	    }
-          gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+	  else
+	    gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+	  if (tsan_func_exit_call)
+	    gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
 	}
       /* We don't use return block (there is either no return in function or
 	 multiple of them).  So create new basic block with return statement.
 	 */
       else
 	{
-	  gimple ret;
+	  greturn *ret;
 	  if (split_point->split_part_set_retval
 	      && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
 	    {
 	      retval = DECL_RESULT (current_function_decl);
 
+	      if (chkp_function_instrumented_p (current_function_decl)
+		  && BOUNDED_P (retval))
+		retbnd = create_tmp_reg (pointer_bounds_type_node);
+
 	      /* We use temporary register to hold value when aggregate_value_p
 		 is false.  Similarly for DECL_BY_REFERENCE we must avoid extra
 		 copy.  */
 	      if (!aggregate_value_p (retval, TREE_TYPE (current_function_decl))
 		  && !DECL_BY_REFERENCE (retval))
-		retval = create_tmp_reg (TREE_TYPE (retval), NULL);
+		retval = create_tmp_reg (TREE_TYPE (retval));
 	      if (is_gimple_reg (retval))
 		{
 		  /* When returning by reference, there is only one SSA name
 		     assigned to RESULT_DECL (that is pointer to return value).
 		     Look it up or create new one if it is missing.  */
 		  if (DECL_BY_REFERENCE (retval))
-		    {
-		      tree retval_name;
-		      if ((retval_name = gimple_default_def (cfun, retval))
-			  != NULL)
-			retval = retval_name;
-		      else
-			{
-		          retval_name = make_ssa_name (retval,
-						       gimple_build_nop ());
-			  set_default_def (retval, retval_name);
-			  retval = retval_name;
-			}
-		    }
+		    retval = get_or_create_ssa_default_def (cfun, retval);
 		  /* Otherwise produce new SSA name for return value.  */
 		  else
 		    retval = make_ssa_name (retval, call);
@@ -1222,15 +1682,34 @@
 	        gimple_call_set_lhs (call, build_simple_mem_ref (retval));
 	      else
 	        gimple_call_set_lhs (call, retval);
+	      gsi_insert_after (&gsi, call, GSI_NEW_STMT);
 	    }
-          gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+	  else
+	    {
+	      gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+	      if (retval
+		  && is_gimple_reg_type (TREE_TYPE (retval))
+		  && !is_gimple_val (retval))
+		{
+		  gassign *g
+		    = gimple_build_assign (make_ssa_name (TREE_TYPE (retval)),
+					   retval);
+		  retval = gimple_assign_lhs (g);
+		  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+		}
+	    }
+	  /* Build bndret call to obtain returned bounds.  */
+	  if (retbnd)
+	    chkp_insert_retbnd_call (retbnd, retval, &gsi);
+	  if (tsan_func_exit_call)
+	    gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
 	  ret = gimple_build_return (retval);
 	  gsi_insert_after (&gsi, ret, GSI_NEW_STMT);
 	}
     }
   free_dominance_info (CDI_DOMINATORS);
   free_dominance_info (CDI_POST_DOMINATORS);
-  compute_inline_parameters (node);
+  compute_fn_summary (node, true);
 }
 
 /* Execute function splitting pass.  */
@@ -1242,12 +1721,13 @@
   basic_block bb;
   int overall_time = 0, overall_size = 0;
   int todo = 0;
-  struct cgraph_node *node = cgraph_node (current_function_decl);
+  struct cgraph_node *node = cgraph_node::get (current_function_decl);
 
-  if (flags_from_decl_or_type (current_function_decl) & ECF_NORETURN)
+  if (flags_from_decl_or_type (current_function_decl)
+      & (ECF_NORETURN|ECF_MALLOC))
     {
       if (dump_file)
-	fprintf (dump_file, "Not splitting: noreturn function.\n");
+	fprintf (dump_file, "Not splitting: noreturn/malloc function.\n");
       return 0;
     }
   if (MAIN_NAME_P (DECL_NAME (current_function_decl)))
@@ -1258,13 +1738,14 @@
     }
   /* This can be relaxed; function might become inlinable after splitting
      away the uninlinable part.  */
-  if (!node->local.inlinable)
+  if (ipa_fn_summaries
+      && !ipa_fn_summaries->get (node)->inlinable)
     {
       if (dump_file)
 	fprintf (dump_file, "Not splitting: not inlinable.\n");
       return 0;
     }
-  if (node->local.disregard_inline_limits)
+  if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
     {
       if (dump_file)
 	fprintf (dump_file, "Not splitting: disregarding inline limits.\n");
@@ -1295,9 +1776,12 @@
      Note that we are not completely conservative about disqualifying functions
      called once.  It is possible that the caller is called more then once and
      then inlining would still benefit.  */
-  if ((!node->callers || !node->callers->next_caller)
+  if ((!node->callers
+       /* Local functions called once will be completely inlined most of time.  */
+       || (!node->callers->next_caller && node->local.local))
       && !node->address_taken
-      && (!flag_lto || !node->local.externally_visible))
+      && !node->has_aliases_p ()
+      && (!flag_lto || !node->externally_visible))
     {
       if (dump_file)
 	fprintf (dump_file, "Not splitting: not called directly "
@@ -1315,10 +1799,21 @@
       return 0;
     }
 
+  /* We enforce splitting after loop headers when profile info is not
+     available.  */
+  if (profile_status_for_fn (cfun) != PROFILE_READ)
+    mark_dfs_back_edges ();
+
+  /* Initialize bitmap to track forbidden calls.  */
+  forbidden_dominators = BITMAP_ALLOC (NULL);
+  calculate_dominance_info (CDI_DOMINATORS);
+
   /* Compute local info about basic blocks and determine function size/time.  */
-  VEC_safe_grow_cleared (bb_info, heap, bb_info_vec, last_basic_block + 1);
+  bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1);
   memset (&best_split_point, 0, sizeof (best_split_point));
-  FOR_EACH_BB (bb)
+  basic_block return_bb = find_return_bb ();
+  int tsan_exit_found = -1;
+  FOR_EACH_BB_FN (bb, cfun)
     {
       int time = 0;
       int size = 0;
@@ -1330,77 +1825,104 @@
       for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
 	{
 	  int this_time, this_size;
-	  gimple stmt = gsi_stmt (bsi);
+	  gimple *stmt = gsi_stmt (bsi);
 
 	  this_size = estimate_num_insns (stmt, &eni_size_weights);
 	  this_time = estimate_num_insns (stmt, &eni_time_weights) * freq;
 	  size += this_size;
 	  time += this_time;
+	  check_forbidden_calls (stmt);
 
 	  if (dump_file && (dump_flags & TDF_DETAILS))
 	    {
 	      fprintf (dump_file, "  freq:%6i size:%3i time:%3i ",
 		       freq, this_size, this_time);
-	      print_gimple_stmt (dump_file, stmt, 0, 0);
+	      print_gimple_stmt (dump_file, stmt, 0);
+	    }
+
+	  if ((flag_sanitize & SANITIZE_THREAD)
+	      && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
+	    {
+	      /* We handle TSAN_FUNC_EXIT for splitting either in the
+		 return_bb, or in its immediate predecessors.  */
+	      if ((bb != return_bb && !find_edge (bb, return_bb))
+		  || (tsan_exit_found != -1
+		      && tsan_exit_found != (bb != return_bb)))
+		{
+		  if (dump_file)
+		    fprintf (dump_file, "Not splitting: TSAN_FUNC_EXIT"
+			     " in unexpected basic block.\n");
+		  BITMAP_FREE (forbidden_dominators);
+		  bb_info_vec.release ();
+		  return 0;
+		}
+	      tsan_exit_found = bb != return_bb;
 	    }
 	}
       overall_time += time;
       overall_size += size;
-      VEC_index (bb_info, bb_info_vec, bb->index)->time = time;
-      VEC_index (bb_info, bb_info_vec, bb->index)->size = size;
+      bb_info_vec[bb->index].time = time;
+      bb_info_vec[bb->index].size = size;
     }
-  find_split_points (overall_time, overall_size);
+  find_split_points (return_bb, overall_time, overall_size);
   if (best_split_point.split_bbs)
     {
-      split_function (&best_split_point);
+      split_function (return_bb, &best_split_point, tsan_exit_found == 1);
       BITMAP_FREE (best_split_point.ssa_names_to_pass);
       BITMAP_FREE (best_split_point.split_bbs);
       todo = TODO_update_ssa | TODO_cleanup_cfg;
     }
-  VEC_free (bb_info, heap, bb_info_vec);
-  bb_info_vec = NULL;
+  BITMAP_FREE (forbidden_dominators);
+  bb_info_vec.release ();
   return todo;
 }
 
-/* Gate function splitting pass.  When doing profile feedback, we want
-   to execute the pass after profiling is read.  So disable one in 
-   early optimization.  */
+namespace {
+
+const pass_data pass_data_split_functions =
+{
+  GIMPLE_PASS, /* type */
+  "fnsplit", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_IPA_FNSPLIT, /* tv_id */
+  PROP_cfg, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
 
-static bool
-gate_split_functions (void)
+class pass_split_functions : public gimple_opt_pass
 {
+public:
+  pass_split_functions (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_split_functions, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  virtual bool gate (function *);
+  virtual unsigned int execute (function *)
+    {
+      return execute_split_functions ();
+    }
+
+}; // class pass_split_functions
+
+bool
+pass_split_functions::gate (function *)
+{
+  /* When doing profile feedback, we want to execute the pass after profiling
+     is read.  So disable one in early optimization.  */
   return (flag_partial_inlining
 	  && !profile_arc_flag && !flag_branch_probabilities);
 }
 
-struct gimple_opt_pass pass_split_functions =
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_split_functions (gcc::context *ctxt)
 {
- {
-  GIMPLE_PASS,
-  "fnsplit",				/* name */
-  gate_split_functions,			/* gate */
-  execute_split_functions,		/* execute */
-  NULL,					/* sub */
-  NULL,					/* next */
-  0,					/* static_pass_number */
-  TV_IPA_FNSPLIT,			/* tv_id */
-  PROP_cfg,				/* properties_required */
-  0,					/* properties_provided */
-  0,					/* properties_destroyed */
-  0,					/* todo_flags_start */
-  TODO_dump_func			/* todo_flags_finish */
- }
-};
-
-/* Gate feedback driven function splitting pass.
-   We don't need to split when profiling at all, we are producing
-   lousy code anyway.  */
-
-static bool
-gate_feedback_split_functions (void)
-{
-  return (flag_partial_inlining
-	  && flag_branch_probabilities);
+  return new pass_split_functions (ctxt);
 }
 
 /* Execute function splitting pass.  */
@@ -1414,21 +1936,50 @@
   return retval;
 }
 
-struct gimple_opt_pass pass_feedback_split_functions =
+namespace {
+
+const pass_data pass_data_feedback_split_functions =
+{
+  GIMPLE_PASS, /* type */
+  "feedback_fnsplit", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_IPA_FNSPLIT, /* tv_id */
+  PROP_cfg, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_feedback_split_functions : public gimple_opt_pass
 {
- {
-  GIMPLE_PASS,
-  "feedback_fnsplit",			/* name */
-  gate_feedback_split_functions,	/* gate */
-  execute_feedback_split_functions,	/* execute */
-  NULL,					/* sub */
-  NULL,					/* next */
-  0,					/* static_pass_number */
-  TV_IPA_FNSPLIT,			/* tv_id */
-  PROP_cfg,				/* properties_required */
-  0,					/* properties_provided */
-  0,					/* properties_destroyed */
-  0,					/* todo_flags_start */
-  TODO_dump_func			/* todo_flags_finish */
- }
-};
+public:
+  pass_feedback_split_functions (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_feedback_split_functions, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  virtual bool gate (function *);
+  virtual unsigned int execute (function *)
+    {
+      return execute_feedback_split_functions ();
+    }
+
+}; // class pass_feedback_split_functions
+
+bool
+pass_feedback_split_functions::gate (function *)
+{
+  /* We don't need to split when profiling at all, we are producing
+     lousy code anyway.  */
+  return (flag_partial_inlining
+	  && flag_branch_probabilities);
+}
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_feedback_split_functions (gcc::context *ctxt)
+{
+  return new pass_feedback_split_functions (ctxt);
+}