CbC/CbC_gcc: gcc/omp-expand.c comparison

comparison gcc/omp-expand.c @ 146:351920fa3827

merge

author	anatofuz <anatofuz@cr.ie.u-ryukyu.ac.jp>
date	Sun, 01 Mar 2020 16:13:28 +0900
parents	1830386684a0
children

comparison

equal deleted inserted replaced

-:8f4e72ab4e11
+:351920fa3827
 /* Expansion pass for OMP directives.  Outlines regions of certain OMP
 directives to separate functions, converts others into explicit calls to the
 runtime library (libgomp) and so forth
-Copyright (C) 2005-2018 Free Software Foundation, Inc.
+Copyright (C) 2005-2020 Free Software Foundation, Inc.
 This file is part of GCC.
 GCC is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
 #include "splay-tree.h"
 #include "cfgloop.h"
 #include "omp-general.h"
 #include "omp-offload.h"
 #include "tree-cfgcleanup.h"
+#include "alloc-pool.h"
 #include "symbol-summary.h"
 #include "gomp-constants.h"
 #include "gimple-pretty-print.h"
 #include "hsa-common.h"
-#include "debug.h"
 #include "stringpool.h"
 #include "attribs.h"
 /* OMP region information.  Every parallel and workshare
 directive is enclosed between two markers, the OMP_* directive
 /* Schedule modifiers.  */
 unsigned char sched_modifiers;
 /* True if this is a combined parallel+workshare region.  */
 bool is_combined_parallel;
+/* Copy of fd.lastprivate_conditional != 0.  */
+bool has_lastprivate_conditional;
 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
 a depend clause.  */
 gomp_ordered *ord_stmt;
 };
 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
 return true;
 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
+if (gimple_omp_for_kind (ws_stmt) != GF_OMP_FOR_KIND_FOR)
+return false;
 omp_extract_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
 return false;
 presence (SIMD_SCHEDULE).  */
 static tree
 omp_adjust_chunk_size (tree chunk_size, bool simd_schedule)
 {
-if (!simd_schedule)
+if (!simd_schedule || integer_zerop (chunk_size))
 return chunk_size;
 poly_uint64 vf = omp_max_vf ();
 if (known_eq (vf, 1U))
 return chunk_size;
 WS_EXIT_BB -> PAR_EXIT_BB.  */
 par_entry_bb = region->entry;
 par_exit_bb = region->exit;
 ws_entry_bb = region->inner->entry;
 ws_exit_bb = region->inner->exit;
+/* Give up for task reductions on the parallel, while it is implementable,
+adding another big set of APIs or slowing down the normal paths is
+not acceptable.  */
+tree pclauses = gimple_omp_parallel_clauses (last_stmt (par_entry_bb));
+if (omp_find_clause (pclauses, OMP_CLAUSE__REDUCTEMP_))
+return;
 if (single_succ (par_entry_bb) == ws_entry_bb
 && single_succ (ws_exit_bb) == par_exit_bb
 && workshare_safe_to_combine_p (ws_entry_bb)
 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
 	  tree clauses = gimple_omp_for_clauses (ws_stmt);
 	  tree c = omp_find_clause (clauses, OMP_CLAUSE_SCHEDULE);
 	  if (c == NULL
 	      || ((OMP_CLAUSE_SCHEDULE_KIND (c) & OMP_CLAUSE_SCHEDULE_MASK)
 		  == OMP_CLAUSE_SCHEDULE_STATIC)
-	      || omp_find_clause (clauses, OMP_CLAUSE_ORDERED))
+	      || omp_find_clause (clauses, OMP_CLAUSE_ORDERED)
-	    {
+	      || omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_)
-	      region->is_combined_parallel = false;
+	      || ((c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_))
-	      region->inner->is_combined_parallel = false;
+		  && POINTER_TYPE_P (TREE_TYPE (OMP_CLAUSE_DECL (c)))))
-	      return;
+	    return;
-	    }
+	}
-	}
+else if (region->inner->type == GIMPLE_OMP_SECTIONS
+	       && (omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
+				    OMP_CLAUSE__REDUCTEMP_)
+		   || omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
+				       OMP_CLAUSE__CONDTEMP_)))
+	return;
 region->is_combined_parallel = true;
 region->inner->is_combined_parallel = true;
 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
 }
 Together, these actions ensure that the debug info for the outlined
 function will be emitted with the correct lexical scope.  */
 static void
-adjust_context_and_scope (tree entry_block, tree child_fndecl)
+adjust_context_and_scope (struct omp_region *region, tree entry_block,
+			  tree child_fndecl)
 {
+tree parent_fndecl = NULL_TREE;
+gimple *entry_stmt;
+/* OMP expansion expands inner regions before outer ones, so if
+we e.g. have explicit task region nested in parallel region, when
+expanding the task region current_function_decl will be the original
+source function, but we actually want to use as context the child
+function of the parallel.  */
+for (region = region->outer;
+region && parent_fndecl == NULL_TREE; region = region->outer)
+switch (region->type)
+{
+case GIMPLE_OMP_PARALLEL:
+case GIMPLE_OMP_TASK:
+case GIMPLE_OMP_TEAMS:
+	entry_stmt = last_stmt (region->entry);
+	parent_fndecl = gimple_omp_taskreg_child_fn (entry_stmt);
+	break;
+case GIMPLE_OMP_TARGET:
+	entry_stmt = last_stmt (region->entry);
+	parent_fndecl
+	  = gimple_omp_target_child_fn (as_a <gomp_target *> (entry_stmt));
+	break;
+default:
+	break;
+}
+if (parent_fndecl == NULL_TREE)
+parent_fndecl = current_function_decl;
+DECL_CONTEXT (child_fndecl) = parent_fndecl;
 if (entry_block != NULL_TREE && TREE_CODE (entry_block) == BLOCK)
 {
 tree b = BLOCK_SUPERCONTEXT (entry_block);
 if (TREE_CODE (b) == BLOCK)
 {
-	  tree parent_fndecl;
-	  /* Follow supercontext chain until the parent fndecl
-	     is found.  */
-	  for (parent_fndecl = BLOCK_SUPERCONTEXT (b);
-	       TREE_CODE (parent_fndecl) == BLOCK;
-	       parent_fndecl = BLOCK_SUPERCONTEXT (parent_fndecl))
-	    ;
-	  gcc_assert (TREE_CODE (parent_fndecl) == FUNCTION_DECL);
-	  DECL_CONTEXT (child_fndecl) = parent_fndecl;
 	  DECL_CHAIN (child_fndecl) = BLOCK_VARS (b);
 	  BLOCK_VARS (b) = child_fndecl;
 	}
 }
 }
-/* Build the function calls to GOMP_parallel_start etc to actually
+/* Build the function calls to GOMP_parallel etc to actually
 generate the parallel operation.  REGION is the parallel region
 being expanded.  BB is the block where to insert the code.  WS_ARGS
 will be set if this is a call to a combined parallel+workshare
 construct, it contains the list of additional arguments needed by
 the workshare construct.  */
 clauses = gimple_omp_parallel_clauses (entry_stmt);
 /* Determine what flavor of GOMP_parallel we will be
 emitting.  */
 start_ix = BUILT_IN_GOMP_PARALLEL;
-if (is_combined_parallel (region))
+tree rtmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+if (rtmp)
+start_ix = BUILT_IN_GOMP_PARALLEL_REDUCTIONS;
+else if (is_combined_parallel (region))
 {
 switch (region->inner->type)
 	{
 	case GIMPLE_OMP_FOR:
 	  gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
 	  switch (region->inner->sched_kind)
 	    {
 	    case OMP_CLAUSE_SCHEDULE_RUNTIME:
-	      start_ix2 = 3;
+	      /* For lastprivate(conditional:), our implementation
+		 requires monotonic behavior.  */
+	      if (region->inner->has_lastprivate_conditional != 0)
+		start_ix2 = 3;
+	      else if ((region->inner->sched_modifiers
+		       & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0)
+		start_ix2 = 6;
+	      else if ((region->inner->sched_modifiers
+			& OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0)
+		start_ix2 = 7;
+	      else
+		start_ix2 = 3;
 	      break;
 	    case OMP_CLAUSE_SCHEDULE_DYNAMIC:
 	    case OMP_CLAUSE_SCHEDULE_GUIDED:
-	      if (region->inner->sched_modifiers
+	      if ((region->inner->sched_modifiers
-		  & OMP_CLAUSE_SCHEDULE_NONMONOTONIC)
+		   & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
+		  && !region->inner->has_lastprivate_conditional)
 		{
 		  start_ix2 = 3 + region->inner->sched_kind;
 		  break;
 		}
 	      /* FALLTHRU */
 else
 t1 = build_fold_addr_expr (t);
 tree child_fndecl = gimple_omp_parallel_child_fn (entry_stmt);
 t2 = build_fold_addr_expr (child_fndecl);
-adjust_context_and_scope (gimple_block (entry_stmt), child_fndecl);
 vec_alloc (args, 4 + vec_safe_length (ws_args));
 args->quick_push (t2);
 args->quick_push (t1);
 args->quick_push (val);
 if (ws_args)
 args->quick_push (flags);
 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
 			       builtin_decl_explicit (start_ix), args);
+if (rtmp)
+{
+tree type = TREE_TYPE (OMP_CLAUSE_DECL (rtmp));
+t = build2 (MODIFY_EXPR, type, OMP_CLAUSE_DECL (rtmp),
+		  fold_convert (type,
+				fold_convert (pointer_sized_int_node, t)));
+}
 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
 			    false, GSI_CONTINUE_LINKING);
 if (hsa_gen_requested_p ()
 && parallel_needs_hsa_kernel_p (region))
 if (ifc == NULL_TREE)
 	iflags |= GOMP_TASK_FLAG_IF;
 if (omp_find_clause (tclauses, OMP_CLAUSE_NOGROUP))
 	iflags |= GOMP_TASK_FLAG_NOGROUP;
 ull = fd.iter_type == long_long_unsigned_type_node;
+if (omp_find_clause (clauses, OMP_CLAUSE_REDUCTION))
+	iflags |= GOMP_TASK_FLAG_REDUCTION;
 }
 else if (priority)
 iflags |= GOMP_TASK_FLAG_PRIORITY;
 tree flags = build_int_cst (unsigned_type_node, iflags);
 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
 			 9, t1, t2, t3,
 			 gimple_omp_task_arg_size (entry_stmt),
 			 gimple_omp_task_arg_align (entry_stmt), cond, flags,
 			 depend, priority);
+force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
+			    false, GSI_CONTINUE_LINKING);
+}
+/* Build the function call to GOMP_taskwait_depend to actually
+generate the taskwait operation.  BB is the block where to insert the
+code.  */
+static void
+expand_taskwait_call (basic_block bb, gomp_task *entry_stmt)
+{
+tree clauses = gimple_omp_task_clauses (entry_stmt);
+tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
+if (depend == NULL_TREE)
+return;
+depend = OMP_CLAUSE_DECL (depend);
+gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
+tree t
+= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASKWAIT_DEPEND),
+		       1, depend);
+force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
+			    false, GSI_CONTINUE_LINKING);
+}
+/* Build the function call to GOMP_teams_reg to actually
+generate the host teams operation.  REGION is the teams region
+being expanded.  BB is the block where to insert the code.  */
+static void
+expand_teams_call (basic_block bb, gomp_teams *entry_stmt)
+{
+tree clauses = gimple_omp_teams_clauses (entry_stmt);
+tree num_teams = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
+if (num_teams == NULL_TREE)
+num_teams = build_int_cst (unsigned_type_node, 0);
+else
+{
+num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
+num_teams = fold_convert (unsigned_type_node, num_teams);
+}
+tree thread_limit = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
+if (thread_limit == NULL_TREE)
+thread_limit = build_int_cst (unsigned_type_node, 0);
+else
+{
+thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
+thread_limit = fold_convert (unsigned_type_node, thread_limit);
+}
+gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
+tree t = gimple_omp_teams_data_arg (entry_stmt), t1;
+if (t == NULL)
+t1 = null_pointer_node;
+else
+t1 = build_fold_addr_expr (t);
+tree child_fndecl = gimple_omp_teams_child_fn (entry_stmt);
+tree t2 = build_fold_addr_expr (child_fndecl);
+vec<tree, va_gc> *args;
+vec_alloc (args, 5);
+args->quick_push (t2);
+args->quick_push (t1);
+args->quick_push (num_teams);
+args->quick_push (thread_limit);
+/* For future extensibility.  */
+args->quick_push (build_zero_cst (unsigned_type_node));
+t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
+			       builtin_decl_explicit (BUILT_IN_GOMP_TEAMS_REG),
+			       args);
 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
 			    false, GSI_CONTINUE_LINKING);
 }
 gimple *entry_stmt, *stmt;
 edge e;
 vec<tree, va_gc> *ws_args;
 entry_stmt = last_stmt (region->entry);
+if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
+&& gimple_omp_task_taskwait_p (entry_stmt))
+{
+new_bb = region->entry;
+gsi = gsi_last_nondebug_bb (region->entry);
+gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
+gsi_remove (&gsi, true);
+expand_taskwait_call (new_bb, as_a <gomp_task *> (entry_stmt));
+return;
+}
 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
 entry_bb = region->entry;
 if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK)
 entry_succ_e = single_succ_edge (entry_bb);
 gsi = gsi_last_nondebug_bb (entry_bb);
 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
-		  || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
+		  || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK
+		  || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TEAMS);
 gsi_remove (&gsi, true);
 new_bb = entry_bb;
 if (exit_bb)
 	{
 		  /* We're ignore the subcode because we're
 		     effectively doing a STRIP_NOPS.  */
 		  if (TREE_CODE (arg) == ADDR_EXPR
-		      && TREE_OPERAND (arg, 0)
+		      && (TREE_OPERAND (arg, 0)
-			== gimple_omp_taskreg_data_arg (entry_stmt))
+			  == gimple_omp_taskreg_data_arg (entry_stmt)))
 		    {
 		      parcopy_stmt = stmt;
 		      break;
 		    }
 		}
 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
 	 so that it can be moved to the child function.  */
 gsi = gsi_last_nondebug_bb (entry_bb);
 stmt = gsi_stmt (gsi);
 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
-			   || gimple_code (stmt) == GIMPLE_OMP_TASK));
+			   || gimple_code (stmt) == GIMPLE_OMP_TASK
+			   || gimple_code (stmt) == GIMPLE_OMP_TEAMS));
 e = split_block (entry_bb, stmt);
 gsi_remove (&gsi, true);
 entry_bb = e->dest;
 edge e2 = NULL;
-if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
+if (gimple_code (entry_stmt) != GIMPLE_OMP_TASK)
 	single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
 else
 	{
 	  e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL);
 	  gcc_assert (e2->dest == region->exit);
 	  block = NULL_TREE;
 	}
 else
 	block = gimple_block (entry_stmt);
-/* Make sure to generate early debug for the function before
-outlining anything.  */
-if (! gimple_in_ssa_p (cfun))
-	(*debug_hooks->early_global_decl) (cfun->decl);
 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
 if (exit_bb)
 	single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
 if (e2)
 	{
 	  dump_function_header (dump_file, child_fn, dump_flags);
 	  dump_function_to_file (child_fn, dump_file, dump_flags);
 	}
 }
+adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
 expand_parallel_call (region, new_bb,
 			  as_a <gomp_parallel *> (entry_stmt), ws_args);
+else if (gimple_code (entry_stmt) == GIMPLE_OMP_TEAMS)
+expand_teams_call (new_bb, as_a <gomp_teams *> (entry_stmt));
 else
 expand_task_call (region, new_bb, as_a <gomp_task *> (entry_stmt));
 if (gimple_in_ssa_p (cfun))
 update_ssa (TODO_update_ssa_only_virtuals);
 }
 	    v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
 					  false, GSI_CONTINUE_LINKING);
 	  t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
 	  stmt = gimple_build_cond_empty (t);
 	  gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
+	  if (walk_tree (gimple_cond_lhs_ptr (as_a <gcond *> (stmt)),
+			 expand_omp_regimplify_p, NULL, NULL)
+	      || walk_tree (gimple_cond_rhs_ptr (as_a <gcond *> (stmt)),
+			    expand_omp_regimplify_p, NULL, NULL))
+	    gimple_regimplify_operands (stmt, &gsi);
 	  e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
 	  e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
 	}
 else
 	make_edge (bb, body_bb, EDGE_FALLTHRU);
 	      if (TREE_CODE (step) != INTEGER_CST)
 		break;
 	      forward = tree_int_cst_sgn (step) != -1;
 	    }
 	  if (forward ^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
-	    warning_at (loc, 0, "%<depend(sink)%> clause waiting for "
+	    warning_at (loc, 0, "%<depend%> clause with %<sink%> modifier "
-				"lexically later iteration");
+				"waiting for lexically later iteration");
 	  break;
 	}
 deps = TREE_CHAIN (deps);
 }
 /* If all offsets corresponding to the collapsed loops are zero,
 				 orig_off ? orig_off : off, s);
 	  t = fold_build2_loc (loc, EQ_EXPR, boolean_type_node, t,
 			       build_int_cst (itype, 0));
 	  if (integer_zerop (t) && !warned_step)
 	    {
-	      warning_at (loc, 0, "%<depend(sink)%> refers to iteration never "
+	      warning_at (loc, 0, "%<depend%> clause with %<sink%> modifier "
-				  "in the iteration space");
+				  "refers to iteration never in the iteration "
+				  "space");
 	      warned_step = true;
 	    }
 	  cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node,
 				  cond, t);
 	}
 set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb);
 set_immediate_dominator (CDI_DOMINATORS, new_body, new_header);
 if (e2)
 	{
-	  struct loop *loop = alloc_loop ();
+	  class loop *loop = alloc_loop ();
 	  loop->header = new_header;
 	  loop->latch = e2->src;
 	  add_loop (loop, body_bb->loop_father);
 	}
 }
 static void
 expand_omp_for_generic (struct omp_region *region,
 			struct omp_for_data *fd,
 			enum built_in_function start_fn,
 			enum built_in_function next_fn,
+			tree sched_arg,
 			gimple *inner_stmt)
 {
 tree type, istart0, iend0, iend;
 tree t, vmain, vback, bias = NULL_TREE;
 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
 gsi = gsi_last_nondebug_bb (entry_bb);
 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
 if (fd->ordered
-&& omp_find_clause (gimple_omp_for_clauses (gsi_stmt (gsi)),
+&& omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
 			  OMP_CLAUSE_LASTPRIVATE))
 ordered_lastprivate = false;
+tree reductions = NULL_TREE;
+tree mem = NULL_TREE, cond_var = NULL_TREE, condtemp = NULL_TREE;
+tree memv = NULL_TREE;
+if (fd->lastprivate_conditional)
+{
+tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
+				OMP_CLAUSE__CONDTEMP_);
+if (fd->have_pointer_condtemp)
+	condtemp = OMP_CLAUSE_DECL (c);
+c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
+cond_var = OMP_CLAUSE_DECL (c);
+}
+if (sched_arg)
+{
+if (fd->have_reductemp)
+	{
+	  tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
+				    OMP_CLAUSE__REDUCTEMP_);
+	  reductions = OMP_CLAUSE_DECL (c);
+	  gcc_assert (TREE_CODE (reductions) == SSA_NAME);
+	  gimple *g = SSA_NAME_DEF_STMT (reductions);
+	  reductions = gimple_assign_rhs1 (g);
+	  OMP_CLAUSE_DECL (c) = reductions;
+	  entry_bb = gimple_bb (g);
+	  edge e = split_block (entry_bb, g);
+	  if (region->entry == entry_bb)
+	    region->entry = e->dest;
+	  gsi = gsi_last_bb (entry_bb);
+	}
+else
+	reductions = null_pointer_node;
+if (fd->have_pointer_condtemp)
+	{
+	  tree type = TREE_TYPE (condtemp);
+	  memv = create_tmp_var (type);
+	  TREE_ADDRESSABLE (memv) = 1;
+	  unsigned HOST_WIDE_INT sz
+	    = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
+	  sz *= fd->lastprivate_conditional;
+	  expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
+				   false);
+	  mem = build_fold_addr_expr (memv);
+	}
+else
+	mem = null_pointer_node;
+}
 if (fd->collapse > 1 || fd->ordered)
 {
 int first_zero_iter1 = -1, first_zero_iter2 = -1;
 basic_block zero_iter1_bb = NULL, zero_iter2_bb = NULL, l2_dom_bb = NULL;
 	{
 	  if (fd->chunk_size)
 	    {
 	      t = fold_convert (fd->iter_type, fd->chunk_size);
 	      t = omp_adjust_chunk_size (t, fd->simd_schedule);
-	      if (fd->ordered)
+	      if (sched_arg)
+		{
+		  if (fd->ordered)
+		    t = build_call_expr (builtin_decl_explicit (start_fn),
+					 8, t0, t1, sched_arg, t, t3, t4,
+					 reductions, mem);
+		  else
+		    t = build_call_expr (builtin_decl_explicit (start_fn),
+					 9, t0, t1, t2, sched_arg, t, t3, t4,
+					 reductions, mem);
+		}
+	      else if (fd->ordered)
 		t = build_call_expr (builtin_decl_explicit (start_fn),
 				     5, t0, t1, t, t3, t4);
 	      else
 		t = build_call_expr (builtin_decl_explicit (start_fn),
 				     6, t0, t1, t2, t, t3, t4);
 	  if (fd->chunk_size)
 	    {
 	      tree bfn_decl = builtin_decl_explicit (start_fn);
 	      t = fold_convert (fd->iter_type, fd->chunk_size);
 	      t = omp_adjust_chunk_size (t, fd->simd_schedule);
-	      t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
+	      if (sched_arg)
+		t = build_call_expr (bfn_decl, 10, t5, t0, t1, t2, sched_arg,
+				     t, t3, t4, reductions, mem);
+	      else
+		t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
 	    }
 	  else
 	    t = build_call_expr (builtin_decl_explicit (start_fn),
 				 6, t5, t0, t1, t2, t3, t4);
 	}
 		     t, build_int_cst (TREE_TYPE (t), 0));
 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
 				true, GSI_SAME_STMT);
 if (arr && !TREE_STATIC (arr))
 {
-tree clobber = build_constructor (TREE_TYPE (arr), NULL);
+tree clobber = build_clobber (TREE_TYPE (arr));
-TREE_THIS_VOLATILE (clobber) = 1;
 gsi_insert_before (&gsi, gimple_build_assign (arr, clobber),
 			 GSI_SAME_STMT);
+}
+if (fd->have_pointer_condtemp)
+expand_omp_build_assign (&gsi, condtemp, memv, false);
+if (fd->have_reductemp)
+{
+gimple *g = gsi_stmt (gsi);
+gsi_remove (&gsi, true);
+release_ssa_name (gimple_assign_lhs (g));
+entry_bb = region->entry;
+gsi = gsi_last_nondebug_bb (entry_bb);
+gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
 }
 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
 /* Remove the GIMPLE_OMP_FOR statement.  */
 gsi_remove (&gsi, true);
 				DECL_P (startvar)
 				&& TREE_ADDRESSABLE (startvar),
 				NULL_TREE, false, GSI_CONTINUE_LINKING);
 assign_stmt = gimple_build_assign (startvar, t);
 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+if (cond_var)
+{
+tree itype = TREE_TYPE (cond_var);
+/* For lastprivate(conditional:) itervar, we need some iteration
+	 counter that starts at unsigned non-zero and increases.
+	 Prefer as few IVs as possible, so if we can use startvar
+	 itself, use that, or startvar + constant (those would be
+	 incremented with step), and as last resort use the s0 + 1
+	 incremented by 1.  */
+if ((fd->ordered && fd->collapse == 1)
+	  || bias
+	  || POINTER_TYPE_P (type)
+	  || TREE_CODE (fd->loop.n1) != INTEGER_CST
+	  || fd->loop.cond_code != LT_EXPR)
+	t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, istart0),
+			 build_int_cst (itype, 1));
+else if (tree_int_cst_sgn (fd->loop.n1) == 1)
+	t = fold_convert (itype, t);
+else
+	{
+	  tree c = fold_convert (itype, fd->loop.n1);
+	  c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
+	  t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
+	}
+t = force_gimple_operand_gsi (&gsi, t, false,
+				    NULL_TREE, false, GSI_CONTINUE_LINKING);
+assign_stmt = gimple_build_assign (cond_var, t);
+gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+}
 t = iend0;
 if (fd->ordered && fd->collapse == 1)
 t = fold_build2 (MULT_EXPR, fd->iter_type, t,
 		     fold_convert (fd->iter_type, fd->loop.step));
 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
 vmain = gimple_omp_continue_control_use (cont_stmt);
 vback = gimple_omp_continue_control_def (cont_stmt);
+if (cond_var)
+	{
+	  tree itype = TREE_TYPE (cond_var);
+	  tree t2;
+	  if ((fd->ordered && fd->collapse == 1)
+	       || bias
+	       || POINTER_TYPE_P (type)
+	       || TREE_CODE (fd->loop.n1) != INTEGER_CST
+	       || fd->loop.cond_code != LT_EXPR)
+	    t2 = build_int_cst (itype, 1);
+	  else
+	    t2 = fold_convert (itype, fd->loop.step);
+	  t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
+	  t2 = force_gimple_operand_gsi (&gsi, t2, false,
+					 NULL_TREE, true, GSI_SAME_STMT);
+	  assign_stmt = gimple_build_assign (cond_var, t2);
+	  gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
+	}
 if (!gimple_omp_for_combined_p (fd->for_stmt))
 	{
 	  if (POINTER_TYPE_P (type))
 	    t = fold_build_pointer_plus (vmain, fd->loop.step);
 	  else
 	  assign_stmt = gimple_build_assign (vback, t);
 	  gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
 	  if (fd->ordered && counts[fd->collapse - 1] == NULL_TREE)
 	    {
+	      tree tem;
 	      if (fd->collapse > 1)
-		t = fd->loop.v;
+		tem = fd->loop.v;
 	      else
 		{
-		  t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
+		  tem = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
-				   fd->loops[0].v, fd->loops[0].n1);
+				     fd->loops[0].v, fd->loops[0].n1);
-		  t = fold_convert (fd->iter_type, t);
+		  tem = fold_convert (fd->iter_type, tem);
 		}
 	      tree aref = build4 (ARRAY_REF, fd->iter_type,
 				  counts[fd->ordered], size_zero_node,
 				  NULL_TREE, NULL_TREE);
-	      t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
+	      tem = force_gimple_operand_gsi (&gsi, tem, true, NULL_TREE,
-					    true, GSI_SAME_STMT);
+					      true, GSI_SAME_STMT);
-	      expand_omp_build_assign (&gsi, aref, t);
+	      expand_omp_build_assign (&gsi, aref, tem);
 	    }
 	  t = build2 (fd->loop.cond_code, boolean_type_node,
 		      DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
 		      iend);
 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
 else
 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
 gcall *call_stmt = gimple_build_call (t, 0);
-if (gimple_omp_return_lhs (gsi_stmt (gsi)))
-gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
-gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
 if (fd->ordered)
 {
 tree arr = counts[fd->ordered];
-tree clobber = build_constructor (TREE_TYPE (arr), NULL);
+tree clobber = build_clobber (TREE_TYPE (arr));
-TREE_THIS_VOLATILE (clobber) = 1;
 gsi_insert_after (&gsi, gimple_build_assign (arr, clobber),
 			GSI_SAME_STMT);
 }
+if (gimple_omp_return_lhs (gsi_stmt (gsi)))
+{
+gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
+if (fd->have_reductemp)
+	{
+	  gimple *g = gimple_build_assign (reductions, NOP_EXPR,
+					   gimple_call_lhs (call_stmt));
+	  gsi_insert_after (&gsi, g, GSI_SAME_STMT);
+	}
+}
+gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
 gsi_remove (&gsi, true);
 /* Connect the new blocks.  */
 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
 	     original loop, and move the loop entry value of the inner phi to
 	     the loop entry value of the outer phi.  */
 	  gphi_iterator psi;
 	  for (psi = gsi_start_phis (l3_bb); !gsi_end_p (psi); gsi_next (&psi))
 	    {
-	      source_location locus;
+	      location_t locus;
 	      gphi *nphi;
 	      gphi *exit_phi = psi.phi ();
 	      if (virtual_operand_p (gimple_phi_result (exit_phi)))
 		continue;
 			       recompute_dominator (CDI_DOMINATORS, l1_bb));
 /* We enter expand_omp_for_generic with a loop.  This original loop may
 	 have its own loop struct, or it may be part of an outer loop struct
 	 (which may be the fake loop).  */
-struct loop *outer_loop = entry_bb->loop_father;
+class loop *outer_loop = entry_bb->loop_father;
 bool orig_loop_has_loop_struct = l1_bb->loop_father != outer_loop;
 add_bb_to_loop (l2_bb, outer_loop);
 /* We've added a new loop around the original loop.  Allocate the
 	 corresponding loop struct.  */
-struct loop *new_loop = alloc_loop ();
+class loop *new_loop = alloc_loop ();
 new_loop->header = l0_bb;
 new_loop->latch = l2_bb;
 add_loop (new_loop, outer_loop);
 /* Allocate a loop structure for the original loop unless we already
 	 had one.  */
 if (!orig_loop_has_loop_struct
 	  && !gimple_omp_for_combined_p (fd->for_stmt))
 	{
-	  struct loop *orig_loop = alloc_loop ();
+	  class loop *orig_loop = alloc_loop ();
 	  orig_loop->header = l1_bb;
 	  /* The loop may have multiple latches.  */
 	  add_loop (orig_loop, new_loop);
 	}
 }
+}
+/* Helper function for expand_omp_for_static_nochunk.  If PTR is NULL,
+compute needed allocation size.  If !ALLOC of team allocations,
+if ALLOC of thread allocation.  SZ is the initial needed size for
+other purposes, ALLOC_ALIGN guaranteed alignment of allocation in bytes,
+CNT number of elements of each array, for !ALLOC this is
+omp_get_num_threads (), for ALLOC number of iterations handled by the
+current thread.  If PTR is non-NULL, it is the start of the allocation
+and this routine shall assign to OMP_CLAUSE_DECL (c) of those _scantemp_
+clauses pointers to the corresponding arrays.  */
+static tree
+expand_omp_scantemp_alloc (tree clauses, tree ptr, unsigned HOST_WIDE_INT sz,
+			   unsigned HOST_WIDE_INT alloc_align, tree cnt,
+			   gimple_stmt_iterator *gsi, bool alloc)
+{
+tree eltsz = NULL_TREE;
+unsigned HOST_WIDE_INT preval = 0;
+if (ptr && sz)
+ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
+		       ptr, size_int (sz));
+for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
+	&& !OMP_CLAUSE__SCANTEMP__CONTROL (c)
+	&& (!OMP_CLAUSE__SCANTEMP__ALLOC (c)) != alloc)
+{
+	tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
+	unsigned HOST_WIDE_INT al = TYPE_ALIGN_UNIT (pointee_type);
+	if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
+	  {
+	    unsigned HOST_WIDE_INT szl
+	      = tree_to_uhwi (TYPE_SIZE_UNIT (pointee_type));
+	    szl = least_bit_hwi (szl);
+	    if (szl)
+	      al = MIN (al, szl);
+	  }
+	if (ptr == NULL_TREE)
+	  {
+	    if (eltsz == NULL_TREE)
+	      eltsz = TYPE_SIZE_UNIT (pointee_type);
+	    else
+	      eltsz = size_binop (PLUS_EXPR, eltsz,
+				  TYPE_SIZE_UNIT (pointee_type));
+	  }
+	if (preval == 0 && al <= alloc_align)
+	  {
+	    unsigned HOST_WIDE_INT diff = ROUND_UP (sz, al) - sz;
+	    sz += diff;
+	    if (diff && ptr)
+	      ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
+				 ptr, size_int (diff));
+	  }
+	else if (al > preval)
+	  {
+	    if (ptr)
+	      {
+		ptr = fold_convert (pointer_sized_int_node, ptr);
+		ptr = fold_build2 (PLUS_EXPR, pointer_sized_int_node, ptr,
+				   build_int_cst (pointer_sized_int_node,
+						  al - 1));
+		ptr = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, ptr,
+				   build_int_cst (pointer_sized_int_node,
+						  -(HOST_WIDE_INT) al));
+		ptr = fold_convert (ptr_type_node, ptr);
+	      }
+	    else
+	      sz += al - 1;
+	  }
+	if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
+	  preval = al;
+	else
+	  preval = 1;
+	if (ptr)
+	  {
+	    expand_omp_build_assign (gsi, OMP_CLAUSE_DECL (c), ptr, false);
+	    ptr = OMP_CLAUSE_DECL (c);
+	    ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr,
+			       size_binop (MULT_EXPR, cnt,
+					   TYPE_SIZE_UNIT (pointee_type)));
+	  }
+}
+if (ptr == NULL_TREE)
+{
+eltsz = size_binop (MULT_EXPR, eltsz, cnt);
+if (sz)
+	eltsz = size_binop (PLUS_EXPR, eltsz, size_int (sz));
+return eltsz;
+}
+else
+return ptr;
 }
 /* A subroutine of expand_omp_for.  Generate code for a parallel
 loop with static schedule and no specified chunk size.  Given
 parameters:
 static void
 expand_omp_for_static_nochunk (struct omp_region *region,
 			       struct omp_for_data *fd,
 			       gimple *inner_stmt)
 {
-tree n, q, s0, e0, e, t, tt, nthreads, threadid;
+tree n, q, s0, e0, e, t, tt, nthreads = NULL_TREE, threadid;
 tree type, itype, vmain, vback;
 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
 basic_block body_bb, cont_bb, collapse_bb = NULL;
-basic_block fin_bb;
+basic_block fin_bb, fourth_bb = NULL, fifth_bb = NULL, sixth_bb = NULL;
-gimple_stmt_iterator gsi;
+basic_block exit1_bb = NULL, exit2_bb = NULL, exit3_bb = NULL;
+gimple_stmt_iterator gsi, gsip;
 edge ep;
 bool broken_loop = region->cont == NULL;
 tree *counts = NULL;
 tree n1, n2, step;
+tree reductions = NULL_TREE;
+tree cond_var = NULL_TREE, condtemp = NULL_TREE;
 itype = type = TREE_TYPE (fd->loop.v);
 if (POINTER_TYPE_P (type))
 itype = signed_type_for (type);
 exit_bb = region->exit;
 /* Iteration space partitioning goes in ENTRY_BB.  */
 gsi = gsi_last_nondebug_bb (entry_bb);
 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
+gsip = gsi;
+gsi_prev (&gsip);
 if (fd->collapse > 1)
 {
 int first_zero_iter = -1, dummy = -1;
 basic_block l2_dom_bb = NULL, dummy_bb = NULL;
 				     true, GSI_SAME_STMT);
 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
 				     true, GSI_SAME_STMT);
 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
-						 NULL_TREE, NULL_TREE);
+					    NULL_TREE, NULL_TREE);
 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
 		     expand_omp_regimplify_p, NULL, NULL)
 	  || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
 			expand_omp_regimplify_p, NULL, NULL))
 	    }
 	}
 gsi = gsi_last_bb (entry_bb);
 }
+if (fd->lastprivate_conditional)
+{
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
+if (fd->have_pointer_condtemp)
+	condtemp = OMP_CLAUSE_DECL (c);
+c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
+cond_var = OMP_CLAUSE_DECL (c);
+}
+if (fd->have_reductemp
+/* For scan, we don't want to reinitialize condtemp before the
+	 second loop.  */
+|| (fd->have_pointer_condtemp && !fd->have_scantemp)
+|| fd->have_nonctrl_scantemp)
+{
+tree t1 = build_int_cst (long_integer_type_node, 0);
+tree t2 = build_int_cst (long_integer_type_node, 1);
+tree t3 = build_int_cstu (long_integer_type_node,
+				(HOST_WIDE_INT_1U << 31) + 1);
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+gimple_stmt_iterator gsi2 = gsi_none ();
+gimple *g = NULL;
+tree mem = null_pointer_node, memv = NULL_TREE;
+unsigned HOST_WIDE_INT condtemp_sz = 0;
+unsigned HOST_WIDE_INT alloc_align = 0;
+if (fd->have_reductemp)
+	{
+	  gcc_assert (!fd->have_nonctrl_scantemp);
+	  tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+	  reductions = OMP_CLAUSE_DECL (c);
+	  gcc_assert (TREE_CODE (reductions) == SSA_NAME);
+	  g = SSA_NAME_DEF_STMT (reductions);
+	  reductions = gimple_assign_rhs1 (g);
+	  OMP_CLAUSE_DECL (c) = reductions;
+	  gsi2 = gsi_for_stmt (g);
+	}
+else
+	{
+	  if (gsi_end_p (gsip))
+	    gsi2 = gsi_after_labels (region->entry);
+	  else
+	    gsi2 = gsip;
+	  reductions = null_pointer_node;
+	}
+if (fd->have_pointer_condtemp || fd->have_nonctrl_scantemp)
+	{
+	  tree type;
+	  if (fd->have_pointer_condtemp)
+	    type = TREE_TYPE (condtemp);
+	  else
+	    type = ptr_type_node;
+	  memv = create_tmp_var (type);
+	  TREE_ADDRESSABLE (memv) = 1;
+	  unsigned HOST_WIDE_INT sz = 0;
+	  tree size = NULL_TREE;
+	  if (fd->have_pointer_condtemp)
+	    {
+	      sz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
+	      sz *= fd->lastprivate_conditional;
+	      condtemp_sz = sz;
+	    }
+	  if (fd->have_nonctrl_scantemp)
+	    {
+	      nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
+	      gimple *g = gimple_build_call (nthreads, 0);
+	      nthreads = create_tmp_var (integer_type_node);
+	      gimple_call_set_lhs (g, nthreads);
+	      gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
+	      nthreads = fold_convert (sizetype, nthreads);
+	      alloc_align = TYPE_ALIGN_UNIT (long_long_integer_type_node);
+	      size = expand_omp_scantemp_alloc (clauses, NULL_TREE, sz,
+						alloc_align, nthreads, NULL,
+						false);
+	      size = fold_convert (type, size);
+	    }
+	  else
+	    size = build_int_cst (type, sz);
+	  expand_omp_build_assign (&gsi2, memv, size, false);
+	  mem = build_fold_addr_expr (memv);
+	}
+tree t
+	= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
+			   9, t1, t2, t2, t3, t1, null_pointer_node,
+			   null_pointer_node, reductions, mem);
+force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
+				true, GSI_SAME_STMT);
+if (fd->have_pointer_condtemp)
+	expand_omp_build_assign (&gsi2, condtemp, memv, false);
+if (fd->have_nonctrl_scantemp)
+	{
+	  tree ptr = fd->have_pointer_condtemp ? condtemp : memv;
+	  expand_omp_scantemp_alloc (clauses, ptr, condtemp_sz,
+				     alloc_align, nthreads, &gsi2, false);
+	}
+if (fd->have_reductemp)
+	{
+	  gsi_remove (&gsi2, true);
+	  release_ssa_name (gimple_assign_lhs (g));
+	}
+}
 switch (gimple_omp_for_kind (fd->for_stmt))
 {
 case GF_OMP_FOR_KIND_FOR:
 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
 third_bb = split_block (second_bb, assign_stmt)->dest;
 gsi = gsi_last_nondebug_bb (third_bb);
 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
+if (fd->have_nonctrl_scantemp)
+{
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+tree controlp = NULL_TREE, controlb = NULL_TREE;
+for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+	if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
+	    && OMP_CLAUSE__SCANTEMP__CONTROL (c))
+	  {
+	    if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
+	      controlb = OMP_CLAUSE_DECL (c);
+	    else
+	      controlp = OMP_CLAUSE_DECL (c);
+	    if (controlb && controlp)
+	      break;
+	  }
+gcc_assert (controlp && controlb);
+tree cnt = create_tmp_var (sizetype);
+gimple *g = gimple_build_assign (cnt, NOP_EXPR, q);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+unsigned HOST_WIDE_INT alloc_align = TYPE_ALIGN_UNIT (ptr_type_node);
+tree sz = expand_omp_scantemp_alloc (clauses, NULL_TREE, 0,
+					   alloc_align, cnt, NULL, true);
+tree size = create_tmp_var (sizetype);
+expand_omp_build_assign (&gsi, size, sz, false);
+tree cmp = fold_build2 (GT_EXPR, boolean_type_node,
+			      size, size_int (16384));
+expand_omp_build_assign (&gsi, controlb, cmp);
+g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
+			     NULL_TREE, NULL_TREE);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+fourth_bb = split_block (third_bb, g)->dest;
+gsi = gsi_last_nondebug_bb (fourth_bb);
+/* FIXME: Once we have allocators, this should use allocator.  */
+g = gimple_build_call (builtin_decl_explicit (BUILT_IN_MALLOC), 1, size);
+gimple_call_set_lhs (g, controlp);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+expand_omp_scantemp_alloc (clauses, controlp, 0, alloc_align, cnt,
+				 &gsi, true);
+gsi_prev (&gsi);
+g = gsi_stmt (gsi);
+fifth_bb = split_block (fourth_bb, g)->dest;
+gsi = gsi_last_nondebug_bb (fifth_bb);
+g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_SAVE), 0);
+gimple_call_set_lhs (g, controlp);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+tree alloca_decl = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
+for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+	if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
+	    && OMP_CLAUSE__SCANTEMP__ALLOC (c))
+	  {
+	    tree tmp = create_tmp_var (sizetype);
+	    tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
+	    g = gimple_build_assign (tmp, MULT_EXPR, cnt,
+				     TYPE_SIZE_UNIT (pointee_type));
+	    gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	    g = gimple_build_call (alloca_decl, 2, tmp,
+				   size_int (TYPE_ALIGN (pointee_type)));
+	    gimple_call_set_lhs (g, OMP_CLAUSE_DECL (c));
+	    gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	  }
+sixth_bb = split_block (fifth_bb, g)->dest;
+gsi = gsi_last_nondebug_bb (sixth_bb);
+}
 t = build2 (MULT_EXPR, itype, q, threadid);
 t = build2 (PLUS_EXPR, itype, t, tt);
 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
 				DECL_P (startvar)
 				&& TREE_ADDRESSABLE (startvar),
 				NULL_TREE, false, GSI_CONTINUE_LINKING);
 assign_stmt = gimple_build_assign (startvar, t);
 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+if (cond_var)
+{
+tree itype = TREE_TYPE (cond_var);
+/* For lastprivate(conditional:) itervar, we need some iteration
+	 counter that starts at unsigned non-zero and increases.
+	 Prefer as few IVs as possible, so if we can use startvar
+	 itself, use that, or startvar + constant (those would be
+	 incremented with step), and as last resort use the s0 + 1
+	 incremented by 1.  */
+if (POINTER_TYPE_P (type)
+	  || TREE_CODE (n1) != INTEGER_CST
+	  || fd->loop.cond_code != LT_EXPR)
+	t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
+			 build_int_cst (itype, 1));
+else if (tree_int_cst_sgn (n1) == 1)
+	t = fold_convert (itype, t);
+else
+	{
+	  tree c = fold_convert (itype, n1);
+	  c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
+	  t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
+	}
+t = force_gimple_operand_gsi (&gsi, t, false,
+				    NULL_TREE, false, GSI_CONTINUE_LINKING);
+assign_stmt = gimple_build_assign (cond_var, t);
+gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+}
 t = fold_convert (itype, e0);
 t = fold_build2 (MULT_EXPR, itype, t, step);
 if (POINTER_TYPE_P (type))
 {
 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
 vmain = gimple_omp_continue_control_use (cont_stmt);
 vback = gimple_omp_continue_control_def (cont_stmt);
+if (cond_var)
+	{
+	  tree itype = TREE_TYPE (cond_var);
+	  tree t2;
+	  if (POINTER_TYPE_P (type)
+	      || TREE_CODE (n1) != INTEGER_CST
+	      || fd->loop.cond_code != LT_EXPR)
+	    t2 = build_int_cst (itype, 1);
+	  else
+	    t2 = fold_convert (itype, step);
+	  t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
+	  t2 = force_gimple_operand_gsi (&gsi, t2, false,
+					 NULL_TREE, true, GSI_SAME_STMT);
+	  assign_stmt = gimple_build_assign (cond_var, t2);
+	  gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
+	}
 if (!gimple_omp_for_combined_p (fd->for_stmt))
 	{
 	  if (POINTER_TYPE_P (type))
 	    t = fold_build_pointer_plus (vmain, step);
 	  else
 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing.  */
 gsi = gsi_last_nondebug_bb (exit_bb);
 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
 {
 t = gimple_omp_return_lhs (gsi_stmt (gsi));
-gsi_insert_after (&gsi, omp_build_barrier (t), GSI_SAME_STMT);
+if (fd->have_reductemp
+	  || ((fd->have_pointer_condtemp || fd->have_scantemp)
+	      && !fd->have_nonctrl_scantemp))
+	{
+	  tree fn;
+	  if (t)
+	    fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
+	  else
+	    fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
+	  gcall *g = gimple_build_call (fn, 0);
+	  if (t)
+	    {
+	      gimple_call_set_lhs (g, t);
+	      if (fd->have_reductemp)
+		gsi_insert_after (&gsi, gimple_build_assign (reductions,
+							     NOP_EXPR, t),
+				  GSI_SAME_STMT);
+	    }
+	  gsi_insert_after (&gsi, g, GSI_SAME_STMT);
+	}
+else
+	gsi_insert_after (&gsi, omp_build_barrier (t), GSI_SAME_STMT);
+}
+else if ((fd->have_pointer_condtemp || fd->have_scantemp)
+	   && !fd->have_nonctrl_scantemp)
+{
+tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
+gcall *g = gimple_build_call (fn, 0);
+gsi_insert_after (&gsi, g, GSI_SAME_STMT);
+}
+if (fd->have_scantemp && !fd->have_nonctrl_scantemp)
+{
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+tree controlp = NULL_TREE, controlb = NULL_TREE;
+for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+	if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
+	    && OMP_CLAUSE__SCANTEMP__CONTROL (c))
+	  {
+	    if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
+	      controlb = OMP_CLAUSE_DECL (c);
+	    else
+	      controlp = OMP_CLAUSE_DECL (c);
+	    if (controlb && controlp)
+	      break;
+	  }
+gcc_assert (controlp && controlb);
+gimple *g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
+				     NULL_TREE, NULL_TREE);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+exit1_bb = split_block (exit_bb, g)->dest;
+gsi = gsi_after_labels (exit1_bb);
+g = gimple_build_call (builtin_decl_explicit (BUILT_IN_FREE), 1,
+			     controlp);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+exit2_bb = split_block (exit1_bb, g)->dest;
+gsi = gsi_after_labels (exit2_bb);
+g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_RESTORE), 1,
+			     controlp);
+gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+exit3_bb = split_block (exit2_bb, g)->dest;
+gsi = gsi_after_labels (exit3_bb);
 }
 gsi_remove (&gsi, true);
 /* Connect all the blocks.  */
 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
 ep->probability = profile_probability::guessed_always ().apply_scale (3, 4);
 ep = find_edge (entry_bb, second_bb);
 ep->flags = EDGE_TRUE_VALUE;
 ep->probability = profile_probability::guessed_always ().apply_scale (1, 4);
-find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
+if (fourth_bb)
-find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
+{
+ep = make_edge (third_bb, fifth_bb, EDGE_FALSE_VALUE);
+ep->probability
+	= profile_probability::guessed_always ().apply_scale (1, 2);
+ep = find_edge (third_bb, fourth_bb);
+ep->flags = EDGE_TRUE_VALUE;
+ep->probability
+	= profile_probability::guessed_always ().apply_scale (1, 2);
+ep = find_edge (fourth_bb, fifth_bb);
+redirect_edge_and_branch (ep, sixth_bb);
+}
+else
+sixth_bb = third_bb;
+find_edge (sixth_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
+find_edge (sixth_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
+if (exit1_bb)
+{
+ep = make_edge (exit_bb, exit2_bb, EDGE_FALSE_VALUE);
+ep->probability
+	= profile_probability::guessed_always ().apply_scale (1, 2);
+ep = find_edge (exit_bb, exit1_bb);
+ep->flags = EDGE_TRUE_VALUE;
+ep->probability
+	= profile_probability::guessed_always ().apply_scale (1, 2);
+ep = find_edge (exit1_bb, exit2_bb);
+redirect_edge_and_branch (ep, exit3_bb);
+}
 if (!broken_loop)
 {
 ep = find_edge (cont_bb, body_bb);
 if (ep == NULL)
 	= ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
 }
 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
-set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
+if (fourth_bb)
+{
+set_immediate_dominator (CDI_DOMINATORS, fifth_bb, third_bb);
+set_immediate_dominator (CDI_DOMINATORS, sixth_bb, third_bb);
+}
+set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, sixth_bb);
 set_immediate_dominator (CDI_DOMINATORS, body_bb,
 			   recompute_dominator (CDI_DOMINATORS, body_bb));
 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
 			   recompute_dominator (CDI_DOMINATORS, fin_bb));
+if (exit1_bb)
-struct loop *loop = body_bb->loop_father;
+{
+set_immediate_dominator (CDI_DOMINATORS, exit2_bb, exit_bb);
+set_immediate_dominator (CDI_DOMINATORS, exit3_bb, exit_bb);
+}
+class loop *loop = body_bb->loop_father;
 if (loop != entry_bb->loop_father)
 {
 gcc_assert (broken_loop || loop->header == body_bb);
 gcc_assert (broken_loop
 		  || loop->latch == region->cont
 tree n, s0, e0, e, t;
 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
 tree type, itype, vmain, vback, vextra;
 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
-gimple_stmt_iterator gsi;
+gimple_stmt_iterator gsi, gsip;
 edge se;
 bool broken_loop = region->cont == NULL;
 tree *counts = NULL;
 tree n1, n2, step;
+tree reductions = NULL_TREE;
+tree cond_var = NULL_TREE, condtemp = NULL_TREE;
 itype = type = TREE_TYPE (fd->loop.v);
 if (POINTER_TYPE_P (type))
 itype = signed_type_for (type);
 exit_bb = region->exit;
 /* Trip and adjustment setup goes in ENTRY_BB.  */
 gsi = gsi_last_nondebug_bb (entry_bb);
 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
+gsip = gsi;
+gsi_prev (&gsip);
 if (fd->collapse > 1)
 {
 int first_zero_iter = -1, dummy = -1;
 basic_block l2_dom_bb = NULL, dummy_bb = NULL;
 	    }
 	}
 gsi = gsi_last_bb (entry_bb);
 }
+if (fd->lastprivate_conditional)
+{
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
+if (fd->have_pointer_condtemp)
+	condtemp = OMP_CLAUSE_DECL (c);
+c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
+cond_var = OMP_CLAUSE_DECL (c);
+}
+if (fd->have_reductemp || fd->have_pointer_condtemp)
+{
+tree t1 = build_int_cst (long_integer_type_node, 0);
+tree t2 = build_int_cst (long_integer_type_node, 1);
+tree t3 = build_int_cstu (long_integer_type_node,
+				(HOST_WIDE_INT_1U << 31) + 1);
+tree clauses = gimple_omp_for_clauses (fd->for_stmt);
+gimple_stmt_iterator gsi2 = gsi_none ();
+gimple *g = NULL;
+tree mem = null_pointer_node, memv = NULL_TREE;
+if (fd->have_reductemp)
+	{
+	  tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+	  reductions = OMP_CLAUSE_DECL (c);
+	  gcc_assert (TREE_CODE (reductions) == SSA_NAME);
+	  g = SSA_NAME_DEF_STMT (reductions);
+	  reductions = gimple_assign_rhs1 (g);
+	  OMP_CLAUSE_DECL (c) = reductions;
+	  gsi2 = gsi_for_stmt (g);
+	}
+else
+	{
+	  if (gsi_end_p (gsip))
+	    gsi2 = gsi_after_labels (region->entry);
+	  else
+	    gsi2 = gsip;
+	  reductions = null_pointer_node;
+	}
+if (fd->have_pointer_condtemp)
+	{
+	  tree type = TREE_TYPE (condtemp);
+	  memv = create_tmp_var (type);
+	  TREE_ADDRESSABLE (memv) = 1;
+	  unsigned HOST_WIDE_INT sz
+	    = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
+	  sz *= fd->lastprivate_conditional;
+	  expand_omp_build_assign (&gsi2, memv, build_int_cst (type, sz),
+				   false);
+	  mem = build_fold_addr_expr (memv);
+	}
+tree t
+	= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
+			   9, t1, t2, t2, t3, t1, null_pointer_node,
+			   null_pointer_node, reductions, mem);
+force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
+				true, GSI_SAME_STMT);
+if (fd->have_pointer_condtemp)
+	expand_omp_build_assign (&gsi2, condtemp, memv, false);
+if (fd->have_reductemp)
+	{
+	  gsi_remove (&gsi2, true);
+	  release_ssa_name (gimple_assign_lhs (g));
+	}
+}
 switch (gimple_omp_for_kind (fd->for_stmt))
 {
 case GF_OMP_FOR_KIND_FOR:
 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
 				DECL_P (startvar)
 				&& TREE_ADDRESSABLE (startvar),
 				NULL_TREE, false, GSI_CONTINUE_LINKING);
 assign_stmt = gimple_build_assign (startvar, t);
 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+if (cond_var)
+{
+tree itype = TREE_TYPE (cond_var);
+/* For lastprivate(conditional:) itervar, we need some iteration
+	 counter that starts at unsigned non-zero and increases.
+	 Prefer as few IVs as possible, so if we can use startvar
+	 itself, use that, or startvar + constant (those would be
+	 incremented with step), and as last resort use the s0 + 1
+	 incremented by 1.  */
+if (POINTER_TYPE_P (type)
+	  || TREE_CODE (n1) != INTEGER_CST
+	  || fd->loop.cond_code != LT_EXPR)
+	t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
+			 build_int_cst (itype, 1));
+else if (tree_int_cst_sgn (n1) == 1)
+	t = fold_convert (itype, t);
+else
+	{
+	  tree c = fold_convert (itype, n1);
+	  c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
+	  t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
+	}
+t = force_gimple_operand_gsi (&gsi, t, false,
+				    NULL_TREE, false, GSI_CONTINUE_LINKING);
+assign_stmt = gimple_build_assign (cond_var, t);
+gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
+}
 t = fold_convert (itype, e0);
 t = fold_build2 (MULT_EXPR, itype, t, step);
 if (POINTER_TYPE_P (type))
 {
 gsi = gsi_last_nondebug_bb (cont_bb);
 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
 vmain = gimple_omp_continue_control_use (cont_stmt);
 vback = gimple_omp_continue_control_def (cont_stmt);
+if (cond_var)
+	{
+	  tree itype = TREE_TYPE (cond_var);
+	  tree t2;
+	  if (POINTER_TYPE_P (type)
+	      || TREE_CODE (n1) != INTEGER_CST
+	      || fd->loop.cond_code != LT_EXPR)
+	    t2 = build_int_cst (itype, 1);
+	  else
+	    t2 = fold_convert (itype, step);
+	  t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
+	  t2 = force_gimple_operand_gsi (&gsi, t2, false,
+					 NULL_TREE, true, GSI_SAME_STMT);
+	  assign_stmt = gimple_build_assign (cond_var, t2);
+	  gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
+	}
 if (!gimple_omp_for_combined_p (fd->for_stmt))
 	{
 	  if (POINTER_TYPE_P (type))
 	    t = fold_build_pointer_plus (vmain, step);
 	  else
 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing.  */
 gsi = gsi_last_nondebug_bb (exit_bb);
 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
 {
 t = gimple_omp_return_lhs (gsi_stmt (gsi));
-gsi_insert_after (&gsi, omp_build_barrier (t), GSI_SAME_STMT);
+if (fd->have_reductemp || fd->have_pointer_condtemp)
+	{
+	  tree fn;
+	  if (t)
+	    fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
+	  else
+	    fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
+	  gcall *g = gimple_build_call (fn, 0);
+	  if (t)
+	    {
+	      gimple_call_set_lhs (g, t);
+	      if (fd->have_reductemp)
+		gsi_insert_after (&gsi, gimple_build_assign (reductions,
+							     NOP_EXPR, t),
+				  GSI_SAME_STMT);
+	    }
+	  gsi_insert_after (&gsi, g, GSI_SAME_STMT);
+	}
+else
+	gsi_insert_after (&gsi, omp_build_barrier (t), GSI_SAME_STMT);
+}
+else if (fd->have_pointer_condtemp)
+{
+tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
+gcall *g = gimple_build_call (fn, 0);
+gsi_insert_after (&gsi, g, GSI_SAME_STMT);
 }
 gsi_remove (&gsi, true);
 /* Connect the new blocks.  */
 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
 psi = gsi_start_phis (fin_bb);
 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
 	   gsi_next (&psi), ++i)
 	{
 	  gphi *nphi;
-	  source_location locus;
+	  location_t locus;
 	  phi = psi.phi ();
 	  if (operand_equal_p (gimple_phi_arg_def (phi, 0),
 			       redirect_edge_var_map_def (vm), 0))
 	    continue;
 set_immediate_dominator (CDI_DOMINATORS, body_bb,
 			   recompute_dominator (CDI_DOMINATORS, body_bb));
 if (!broken_loop)
 {
-struct loop *loop = body_bb->loop_father;
+class loop *loop = body_bb->loop_father;
-struct loop *trip_loop = alloc_loop ();
+class loop *trip_loop = alloc_loop ();
 trip_loop->header = iter_part_bb;
 trip_loop->latch = trip_update_bb;
 add_loop (trip_loop, iter_part_bb->loop_father);
 if (loop != entry_bb->loop_father)
 bool broken_loop = region->cont == NULL;
 edge e, ne;
 tree *counts = NULL;
 int i;
 int safelen_int = INT_MAX;
+bool dont_vectorize = false;
 tree safelen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
 				  OMP_CLAUSE_SAFELEN);
 tree simduid = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
 				  OMP_CLAUSE__SIMDUID_);
+tree ifc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
+			      OMP_CLAUSE_IF);
+tree simdlen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
+				  OMP_CLAUSE_SIMDLEN);
+tree condtemp = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
+				   OMP_CLAUSE__CONDTEMP_);
 tree n1, n2;
+tree cond_var = condtemp ? OMP_CLAUSE_DECL (condtemp) : NULL_TREE;
 if (safelen)
 {
 poly_uint64 val;
 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
 	safelen_int = 0;
 else
 	safelen_int = MIN (constant_lower_bound (val), INT_MAX);
 if (safelen_int == 1)
 	safelen_int = 0;
+}
+if ((ifc && integer_zerop (OMP_CLAUSE_IF_EXPR (ifc)))
+|| (simdlen && integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (simdlen))))
+{
+safelen_int = 0;
+dont_vectorize = true;
 }
 type = TREE_TYPE (fd->loop.v);
 entry_bb = region->entry;
 cont_bb = region->cont;
 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
 	    if (POINTER_TYPE_P (itype))
 	      itype = signed_type_for (itype);
 	    t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
 	    expand_omp_build_assign (&gsi, fd->loops[i].v, t);
 	  }
+}
+if (cond_var)
+{
+if (POINTER_TYPE_P (type)
+	  || TREE_CODE (n1) != INTEGER_CST
+	  || fd->loop.cond_code != LT_EXPR
+	  || tree_int_cst_sgn (n1) != 1)
+	expand_omp_build_assign (&gsi, cond_var,
+				 build_one_cst (TREE_TYPE (cond_var)));
+else
+	expand_omp_build_assign (&gsi, cond_var,
+				 fold_convert (TREE_TYPE (cond_var), n1));
 }
 /* Remove the GIMPLE_OMP_FOR statement.  */
 gsi_remove (&gsi, true);
 				  fold_convert (itype, fd->loops[i].n2)),
 			  fd->loops[i].v, t);
 	      expand_omp_build_assign (&gsi, fd->loops[i].v, t);
 	    }
 	}
+if (cond_var)
+	{
+	  if (POINTER_TYPE_P (type)
+	      || TREE_CODE (n1) != INTEGER_CST
+	      || fd->loop.cond_code != LT_EXPR
+	      || tree_int_cst_sgn (n1) != 1)
+	    t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
+			     build_one_cst (TREE_TYPE (cond_var)));
+	  else
+	    t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
+			     fold_convert (TREE_TYPE (cond_var), step));
+	  expand_omp_build_assign (&gsi, cond_var, t);
+	}
 /* Remove GIMPLE_OMP_CONTINUE.  */
 gsi_remove (&gsi, true);
 }
 }
 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
 if (!broken_loop)
 {
-struct loop *loop = alloc_loop ();
+class loop *loop = alloc_loop ();
 loop->header = l1_bb;
 loop->latch = cont_bb;
 add_loop (loop, l1_bb->loop_father);
 loop->safelen = safelen_int;
 if (simduid)
 	   || !global_options_set.x_flag_tree_loop_vectorize)
 	  && flag_tree_loop_optimize
 	  && loop->safelen > 1)
 	{
 	  loop->force_vectorize = true;
+	  if (simdlen && tree_fits_uhwi_p (OMP_CLAUSE_SIMDLEN_EXPR (simdlen)))
+	    {
+	      unsigned HOST_WIDE_INT v
+		= tree_to_uhwi (OMP_CLAUSE_SIMDLEN_EXPR (simdlen));
+	      if (v < INT_MAX && v <= (unsigned HOST_WIDE_INT) loop->safelen)
+		loop->simdlen = v;
+	    }
 	  cfun->has_force_vectorize_loops = true;
 	}
+else if (dont_vectorize)
+	loop->dont_vectorize = true;
 }
 else if (simduid)
 cfun->has_simduid_loops = true;
 }
 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
 			     recompute_dominator (CDI_DOMINATORS, fin_bb));
 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
 {
-struct loop *loop = alloc_loop ();
+class loop *loop = alloc_loop ();
 loop->header = body_bb;
 if (collapse_bb == NULL)
 	loop->latch = cont_bb;
 add_loop (loop, body_bb->loop_father);
 }
 if (cont_bb)
 {
 /* We now have one, two or three nested loops.  Update the loop
 	 structures.  */
-struct loop *parent = entry_bb->loop_father;
+class loop *parent = entry_bb->loop_father;
-struct loop *body = body_bb->loop_father;
+class loop *body = body_bb->loop_father;
 if (chunking)
 	{
-	  struct loop *chunk_loop = alloc_loop ();
+	  class loop *chunk_loop = alloc_loop ();
 	  chunk_loop->header = head_bb;
 	  chunk_loop->latch = bottom_bb;
 	  add_loop (chunk_loop, parent);
 	  parent = chunk_loop;
 	}
 	  parent = NULL;
 	}
 if (parent)
 	{
-	  struct loop *body_loop = alloc_loop ();
+	  class loop *body_loop = alloc_loop ();
 	  body_loop->header = body_bb;
 	  body_loop->latch = cont_bb;
 	  add_loop (body_loop, parent);
 	  if (fd->tiling)
 	    {
 	      /* Insert tiling's element loop.  */
-	      struct loop *inner_loop = alloc_loop ();
+	      class loop *inner_loop = alloc_loop ();
 	      inner_loop->header = elem_body_bb;
 	      inner_loop->latch = elem_cont_bb;
 	      add_loop (inner_loop, body_loop);
 	    }
 	}
 	      * sizeof (struct omp_for_data_loop));
 omp_extract_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
 			&fd, loops);
 region->sched_kind = fd.sched_kind;
 region->sched_modifiers = fd.sched_modifiers;
+region->has_lastprivate_conditional = fd.lastprivate_conditional != 0;
 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
 if (region->cont)
 /* If there isn't a continue then this is a degerate case where
 the introduction of abnormal edges during lowering will prevent
 original loops from being detected.  Fix that up.  */
 loops_state_set (LOOPS_NEED_FIXUP);
-if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
+if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_SIMD)
 expand_omp_simd (region, &fd);
 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
 {
 gcc_assert (!inner_stmt);
 expand_oacc_for (region, &fd);
 	expand_omp_for_static_chunk (region, &fd, inner_stmt);
 }
 else
 {
 int fn_index, start_ix, next_ix;
+unsigned HOST_WIDE_INT sched = 0;
+tree sched_arg = NULL_TREE;
 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
 		  == GF_OMP_FOR_KIND_FOR);
 if (fd.chunk_size == NULL
 	  && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
 	fd.chunk_size = integer_zero_node;
-gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
 switch (fd.sched_kind)
 	{
 	case OMP_CLAUSE_SCHEDULE_RUNTIME:
-	  fn_index = 3;
+	  if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0
+	      && fd.lastprivate_conditional == 0)
+	    {
+	      gcc_assert (!fd.have_ordered);
+	      fn_index = 6;
+	      sched = 4;
+	    }
+	  else if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
+		   && !fd.have_ordered
+		   && fd.lastprivate_conditional == 0)
+	    fn_index = 7;
+	  else
+	    {
+	      fn_index = 3;
+	      sched = (HOST_WIDE_INT_1U << 31);
+	    }
 	  break;
 	case OMP_CLAUSE_SCHEDULE_DYNAMIC:
 	case OMP_CLAUSE_SCHEDULE_GUIDED:
-	  if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_NONMONOTONIC)
+	  if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
-	      && !fd.ordered
+	      && !fd.have_ordered
-	      && !fd.have_ordered)
+	      && fd.lastprivate_conditional == 0)
 	    {
 	      fn_index = 3 + fd.sched_kind;
+	      sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
 	      break;
 	    }
-	  /* FALLTHRU */
+	  fn_index = fd.sched_kind;
+	  sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
+	  sched += (HOST_WIDE_INT_1U << 31);
+	  break;
+	case OMP_CLAUSE_SCHEDULE_STATIC:
+	  gcc_assert (fd.have_ordered);
+	  fn_index = 0;
+	  sched = (HOST_WIDE_INT_1U << 31) + 1;
+	  break;
 	default:
-	  fn_index = fd.sched_kind;
+	  gcc_unreachable ();
-	  break;
 	}
 if (!fd.ordered)
-	fn_index += fd.have_ordered * 6;
+	fn_index += fd.have_ordered * 8;
 if (fd.ordered)
 	start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
 else
 	start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
+if (fd.have_reductemp || fd.have_pointer_condtemp)
+	{
+	  if (fd.ordered)
+	    start_ix = (int)BUILT_IN_GOMP_LOOP_DOACROSS_START;
+	  else if (fd.have_ordered)
+	    start_ix = (int)BUILT_IN_GOMP_LOOP_ORDERED_START;
+	  else
+	    start_ix = (int)BUILT_IN_GOMP_LOOP_START;
+	  sched_arg = build_int_cstu (long_integer_type_node, sched);
+	  if (!fd.chunk_size)
+	    fd.chunk_size = integer_zero_node;
+	}
 if (fd.iter_type == long_long_unsigned_type_node)
 	{
 	  start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
 			- (int)BUILT_IN_GOMP_LOOP_STATIC_START);
 	  next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
 		      - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
 	}
 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
-			      (enum built_in_function) next_ix, inner_stmt);
+			      (enum built_in_function) next_ix, sched_arg,
+			      inner_stmt);
 }
 if (gimple_in_ssa_p (cfun))
 update_ssa (TODO_update_ssa_only_virtuals);
 }
 GIMPLE_OMP_SECTIONS statement.  */
 si = gsi_last_nondebug_bb (entry_bb);
 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
 vin = gimple_omp_sections_control (sections_stmt);
-if (!is_combined_parallel (region))
+tree clauses = gimple_omp_sections_clauses (sections_stmt);
+tree reductmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+tree condtmp = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
+tree cond_var = NULL_TREE;
+if (reductmp || condtmp)
+{
+tree reductions = null_pointer_node, mem = null_pointer_node;
+tree memv = NULL_TREE, condtemp = NULL_TREE;
+gimple_stmt_iterator gsi = gsi_none ();
+gimple *g = NULL;
+if (reductmp)
+	{
+	  reductions = OMP_CLAUSE_DECL (reductmp);
+	  gcc_assert (TREE_CODE (reductions) == SSA_NAME);
+	  g = SSA_NAME_DEF_STMT (reductions);
+	  reductions = gimple_assign_rhs1 (g);
+	  OMP_CLAUSE_DECL (reductmp) = reductions;
+	  gsi = gsi_for_stmt (g);
+	}
+else
+	gsi = si;
+if (condtmp)
+	{
+	  condtemp = OMP_CLAUSE_DECL (condtmp);
+	  tree c = omp_find_clause (OMP_CLAUSE_CHAIN (condtmp),
+				    OMP_CLAUSE__CONDTEMP_);
+	  cond_var = OMP_CLAUSE_DECL (c);
+	  tree type = TREE_TYPE (condtemp);
+	  memv = create_tmp_var (type);
+	  TREE_ADDRESSABLE (memv) = 1;
+	  unsigned cnt = 0;
+	  for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+	    if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+		&& OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
+	      ++cnt;
+	  unsigned HOST_WIDE_INT sz
+	    = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type))) * cnt;
+	  expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
+				   false);
+	  mem = build_fold_addr_expr (memv);
+	}
+t = build_int_cst (unsigned_type_node, len - 1);
+u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS2_START);
+stmt = gimple_build_call (u, 3, t, reductions, mem);
+gimple_call_set_lhs (stmt, vin);
+gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
+if (condtmp)
+	{
+	  expand_omp_build_assign (&gsi, condtemp, memv, false);
+	  tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
+			   vin, build_one_cst (TREE_TYPE (cond_var)));
+	  expand_omp_build_assign (&gsi, cond_var, t, false);
+	}
+if (reductmp)
+	{
+	  gsi_remove (&gsi, true);
+	  release_ssa_name (gimple_assign_lhs (g));
+	}
+}
+else if (!is_combined_parallel (region))
 {
 /* If we are not inside a combined parallel+sections region,
 	 call GOMP_sections_start.  */
 t = build_int_cst (unsigned_type_node, len - 1);
 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
 {
 /* Otherwise, call GOMP_sections_next.  */
 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
 stmt = gimple_build_call (u, 0);
 }
-gimple_call_set_lhs (stmt, vin);
+if (!reductmp && !condtmp)
-gsi_insert_after (&si, stmt, GSI_SAME_STMT);
+{
+gimple_call_set_lhs (stmt, vin);
+gsi_insert_after (&si, stmt, GSI_SAME_STMT);
+}
 gsi_remove (&si, true);
 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
 L0_BB.  */
 switch_si = gsi_last_nondebug_bb (l0_bb);
 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
 stmt = gimple_build_call (bfn_decl, 0);
 gimple_call_set_lhs (stmt, vnext);
-gsi_insert_after (&si, stmt, GSI_SAME_STMT);
+gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+if (cond_var)
+	{
+	  tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
+			   vnext, build_one_cst (TREE_TYPE (cond_var)));
+	  expand_omp_build_assign (&si, cond_var, t, false);
+	}
 gsi_remove (&si, true);
 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
 }
 	      || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
 	      || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
 	      || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
 	      || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
 	      || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
+if (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS
+&& gimple_omp_teams_host (as_a <gomp_teams *> (gsi_stmt (si))))
+{
+expand_omp_taskreg (region);
+return;
+}
 gsi_remove (&si, true);
 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
 if (exit_bb)
 {
 si = gsi_last_nondebug_bb (exit_bb);
 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
 gsi_remove (&si, true);
 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
+}
+}
+/* Translate enum omp_memory_order to enum memmodel.  The two enums
+are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
+is 0.  */
+static enum memmodel
+omp_memory_order_to_memmodel (enum omp_memory_order mo)
+{
+switch (mo)
+{
+case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
+case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
+case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELEASE;
+case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQ_REL;
+case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+default: gcc_unreachable ();
 }
 }
 /* A subroutine of expand_omp_atomic.  Attempt to implement the atomic
 operation as a normal volatile load.  */
 return false;
 type = TREE_TYPE (loaded_val);
 itype = TREE_TYPE (TREE_TYPE (decl));
-call = build_call_expr_loc (loc, decl, 2, addr,
+enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
-			      build_int_cst (NULL,
+tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
-					     gimple_omp_atomic_seq_cst_p (stmt)
+call = build_call_expr_loc (loc, decl, 2, addr, mo);
-					     ? MEMMODEL_SEQ_CST
-					     : MEMMODEL_RELAXED));
 if (!useless_type_conversion_p (type, itype))
 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
 if (exchange && !can_atomic_exchange_p (imode, true))
 return false;
 if (!useless_type_conversion_p (itype, type))
 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
-call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
+enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
-			      build_int_cst (NULL,
+tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
-					     gimple_omp_atomic_seq_cst_p (stmt)
+call = build_call_expr_loc (loc, decl, 3, addr, stored_val, mo);
-					     ? MEMMODEL_SEQ_CST
-					     : MEMMODEL_RELAXED));
 if (exchange)
 {
 if (!useless_type_conversion_p (type, itype))
 	call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
 gimple *stmt;
 location_t loc;
 enum tree_code code;
 bool need_old, need_new;
 machine_mode imode;
-bool seq_cst;
 /* We expect to find the following sequences:
 load_bb:
 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
 gsi_next_nondebug (&gsi);
 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
 return false;
 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
-seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
+enum omp_memory_order omo
+= gimple_omp_atomic_memory_order (last_stmt (load_bb));
+enum memmodel mo = omp_memory_order_to_memmodel (omo);
 gcc_checking_assert (!need_old || !need_new);
 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
 return false;
 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
 It only requires that the operation happen atomically.  Thus we can
 use the RELAXED memory model.  */
 call = build_call_expr_loc (loc, decl, 3, addr,
 			      fold_convert_loc (loc, itype, rhs),
-			      build_int_cst (NULL,
+			      build_int_cst (NULL, mo));
-					     seq_cst ? MEMMODEL_SEQ_CST
-						     : MEMMODEL_RELAXED));
 if (need_old || need_new)
 {
 lhs = need_old ? loaded_val : stored_val;
 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
 }
 /* Remove GIMPLE_OMP_ATOMIC_STORE.  */
 gsi_remove (&si, true);
-struct loop *loop = alloc_loop ();
+class loop *loop = alloc_loop ();
 loop->header = loop_header;
 loop->latch = store_bb;
 add_loop (loop, loop_header->loop_father);
 if (gimple_in_ssa_p (cfun))
 static void
 mark_loops_in_oacc_kernels_region (basic_block region_entry,
 				   basic_block region_exit)
 {
-struct loop *outer = region_entry->loop_father;
+class loop *outer = region_entry->loop_father;
 gcc_assert (region_exit == NULL || outer == region_exit->loop_father);
 /* Don't parallelize the kernels region if it contains more than one outer
 loop.  */
 unsigned int nr_outer_loops = 0;
-struct loop *single_outer = NULL;
+class loop *single_outer = NULL;
-for (struct loop *loop = outer->inner; loop != NULL; loop = loop->next)
+for (class loop *loop = outer->inner; loop != NULL; loop = loop->next)
 {
 gcc_assert (loop_outer (loop) == outer);
 if (!dominated_by_p (CDI_DOMINATORS, loop->header, region_entry))
 	continue;
 single_outer = loop;
 }
 if (nr_outer_loops != 1)
 return;
-for (struct loop *loop = single_outer->inner;
+for (class loop *loop = single_outer->inner;
 loop != NULL;
 loop = loop->inner)
 if (loop->next)
 return;
 /* Mark the loops in the region.  */
-for (struct loop *loop = single_outer; loop != NULL; loop = loop->inner)
+for (class loop *loop = single_outer; loop != NULL; loop = loop->inner)
 loop->in_oacc_kernels_region = true;
 }
 /* Types used to pass grid and wortkgroup sizes to kernel invocation.  */
 gimple_stmt_iterator gsi;
 gomp_target *entry_stmt;
 gimple *stmt;
 edge e;
 bool offloaded, data_region;
+int target_kind;
 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry));
+target_kind = gimple_omp_target_kind (entry_stmt);
 new_bb = region->entry;
 offloaded = is_gimple_omp_offloaded (entry_stmt);
-switch (gimple_omp_target_kind (entry_stmt))
+switch (target_kind)
 {
 case GF_OMP_TARGET_KIND_REGION:
 case GF_OMP_TARGET_KIND_UPDATE:
 case GF_OMP_TARGET_KIND_ENTER_DATA:
 case GF_OMP_TARGET_KIND_EXIT_DATA:
 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
 case GF_OMP_TARGET_KIND_OACC_KERNELS:
+case GF_OMP_TARGET_KIND_OACC_SERIAL:
 case GF_OMP_TARGET_KIND_OACC_UPDATE:
 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
 case GF_OMP_TARGET_KIND_OACC_DECLARE:
 data_region = false;
 break;
 gcc_checking_assert (!gimple_in_ssa_p (cfun));
 entry_bb = region->entry;
 exit_bb = region->exit;
-if (gimple_omp_target_kind (entry_stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
+switch (target_kind)
 {
+case GF_OMP_TARGET_KIND_OACC_KERNELS:
 mark_loops_in_oacc_kernels_region (region->entry, region->exit);
-/* Further down, both OpenACC kernels and OpenACC parallel constructs
+/* Further down, all OpenACC compute constructs will be mapped to
-	 will be mappted to BUILT_IN_GOACC_PARALLEL, and to distinguish the
+	 BUILT_IN_GOACC_PARALLEL, and to distinguish between them, there
-	 two, there is an "oacc kernels" attribute set for OpenACC kernels.  */
+	 is an "oacc kernels" attribute set for OpenACC kernels.  */
 DECL_ATTRIBUTES (child_fn)
 	= tree_cons (get_identifier ("oacc kernels"),
 		     NULL_TREE, DECL_ATTRIBUTES (child_fn));
+break;
+case GF_OMP_TARGET_KIND_OACC_SERIAL:
+/* Further down, all OpenACC compute constructs will be mapped to
+	 BUILT_IN_GOACC_PARALLEL, and to distinguish between them, there
+	 is an "oacc serial" attribute set for OpenACC serial.  */
+DECL_ATTRIBUTES (child_fn)
+	= tree_cons (get_identifier ("oacc serial"),
+		     NULL_TREE, DECL_ATTRIBUTES (child_fn));
+break;
+default:
+break;
 }
 if (offloaded)
 {
 unsigned srcidx, dstidx, num;
 	  stmt = gimple_build_return (NULL);
 	  gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
 	  gsi_remove (&gsi, true);
 	}
-/* Make sure to generate early debug for the function before
-outlining anything.  */
-if (! gimple_in_ssa_p (cfun))
-	(*debug_hooks->early_global_decl) (cfun->decl);
 /* Move the offloading region into CHILD_CFUN.  */
 block = gimple_block (entry_stmt);
 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
 	{
 	  omp_any_child_fn_dumped = true;
 	  dump_function_header (dump_file, child_fn, dump_flags);
 	  dump_function_to_file (child_fn, dump_file, dump_flags);
 	}
+adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
 }
 /* Emit a library call to launch the offloading region, or do data
 transfers.  */
-tree t1, t2, t3, t4, device, cond, depend, c, clauses;
+tree t1, t2, t3, t4, depend, c, clauses;
 enum built_in_function start_ix;
-location_t clause_loc;
 unsigned int flags_i = 0;
 switch (gimple_omp_target_kind (entry_stmt))
 {
 case GF_OMP_TARGET_KIND_REGION:
 break;
 case GF_OMP_TARGET_KIND_EXIT_DATA:
 start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
 flags_i |= GOMP_TARGET_FLAG_EXIT_DATA;
 break;
+case GF_OMP_TARGET_KIND_OACC_PARALLEL:
 case GF_OMP_TARGET_KIND_OACC_KERNELS:
-case GF_OMP_TARGET_KIND_OACC_PARALLEL:
+case GF_OMP_TARGET_KIND_OACC_SERIAL:
 start_ix = BUILT_IN_GOACC_PARALLEL;
 break;
 case GF_OMP_TARGET_KIND_OACC_DATA:
 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
 start_ix = BUILT_IN_GOACC_DATA_START;
 gcc_unreachable ();
 }
 clauses = gimple_omp_target_clauses (entry_stmt);
-/* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
+tree device = NULL_TREE;
-library choose) and there is no conditional.  */
+location_t device_loc = UNKNOWN_LOCATION;
-cond = NULL_TREE;
+tree goacc_flags = NULL_TREE;
-device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
+if (is_gimple_omp_oacc (entry_stmt))
+{
+/* By default, no GOACC_FLAGs are set.  */
+goacc_flags = integer_zero_node;
+}
+else
+{
+c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE);
+if (c)
+	{
+	  device = OMP_CLAUSE_DEVICE_ID (c);
+	  device_loc = OMP_CLAUSE_LOCATION (c);
+	}
+else
+	{
+	  /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
+	     library choose).  */
+	  device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
+	  device_loc = gimple_location (entry_stmt);
+	}
+c = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT);
+if (c)
+	flags_i |= GOMP_TARGET_FLAG_NOWAIT;
+}
+/* By default, there is no conditional.  */
+tree cond = NULL_TREE;
 c = omp_find_clause (clauses, OMP_CLAUSE_IF);
 if (c)
 cond = OMP_CLAUSE_IF_EXPR (c);
+/* If we found the clause 'if (cond)', build:
-c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE);
+OpenACC: goacc_flags = (cond ? goacc_flags : flags | GOACC_FLAG_HOST_FALLBACK)
-if (c)
+OpenMP: device = (cond ? device : GOMP_DEVICE_HOST_FALLBACK) */
-{
-/* Even if we pass it to all library function calls, it is currently only
-	 defined/used for the OpenMP target ones.  */
-gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET
-			   || start_ix == BUILT_IN_GOMP_TARGET_DATA
-			   || start_ix == BUILT_IN_GOMP_TARGET_UPDATE
-			   || start_ix == BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA);
-device = OMP_CLAUSE_DEVICE_ID (c);
-clause_loc = OMP_CLAUSE_LOCATION (c);
-}
-else
-clause_loc = gimple_location (entry_stmt);
-c = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT);
-if (c)
-flags_i |= GOMP_TARGET_FLAG_NOWAIT;
-/* Ensure 'device' is of the correct type.  */
-device = fold_convert_loc (clause_loc, integer_type_node, device);
-/* If we found the clause 'if (cond)', build
-(cond ? device : GOMP_DEVICE_HOST_FALLBACK).  */
 if (cond)
 {
+tree *tp;
+if (is_gimple_omp_oacc (entry_stmt))
+	tp = &goacc_flags;
+else
+	{
+	  /* Ensure 'device' is of the correct type.  */
+	  device = fold_convert_loc (device_loc, integer_type_node, device);
+	  tp = &device;
+	}
 cond = gimple_boolify (cond);
 basic_block cond_bb, then_bb, else_bb;
 edge e;
 tree tmp_var;
-tmp_var = create_tmp_var (TREE_TYPE (device));
+tmp_var = create_tmp_var (TREE_TYPE (*tp));
 if (offloaded)
 	e = split_block_after_labels (new_bb);
 else
 	{
 	  gsi = gsi_last_nondebug_bb (new_bb);
 stmt = gimple_build_cond_empty (cond);
 gsi = gsi_last_bb (cond_bb);
 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
 gsi = gsi_start_bb (then_bb);
-stmt = gimple_build_assign (tmp_var, device);
+stmt = gimple_build_assign (tmp_var, *tp);
 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
 gsi = gsi_start_bb (else_bb);
-stmt = gimple_build_assign (tmp_var,
+if (is_gimple_omp_oacc (entry_stmt))
-				  build_int_cst (integer_type_node,
+	stmt = gimple_build_assign (tmp_var,
-						 GOMP_DEVICE_HOST_FALLBACK));
+				    BIT_IOR_EXPR,
+				    *tp,
+				    build_int_cst (integer_type_node,
+						   GOACC_FLAG_HOST_FALLBACK));
+else
+	stmt = gimple_build_assign (tmp_var,
+				    build_int_cst (integer_type_node,
+						   GOMP_DEVICE_HOST_FALLBACK));
 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
 add_bb_to_loop (then_bb, cond_bb->loop_father);
 add_bb_to_loop (else_bb, cond_bb->loop_father);
 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
-device = tmp_var;
+*tp = tmp_var;
 gsi = gsi_last_nondebug_bb (new_bb);
 }
 else
 {
 gsi = gsi_last_nondebug_bb (new_bb);
-device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
-					 true, GSI_SAME_STMT);
+if (device != NULL_TREE)
+	device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
+					   true, GSI_SAME_STMT);
 }
 t = gimple_omp_target_data_arg (entry_stmt);
 if (t == NULL)
 {
 gimple *g;
 bool tagging = false;
 /* The maximum number used by any start_ix, without varargs.  */
 auto_vec<tree, 11> args;
-args.quick_push (device);
+if (is_gimple_omp_oacc (entry_stmt))
+{
+tree goacc_flags_m = fold_build1 (GOACC_FLAGS_MARSHAL_OP,
+					TREE_TYPE (goacc_flags), goacc_flags);
+goacc_flags_m = force_gimple_operand_gsi (&gsi, goacc_flags_m, true,
+						NULL_TREE, true,
+						GSI_SAME_STMT);
+args.quick_push (goacc_flags_m);
+}
+else
+args.quick_push (device);
 if (offloaded)
 args.quick_push (build_fold_addr_expr (child_fn));
 args.quick_push (t1);
 args.quick_push (t2);
 args.quick_push (t3);
 args.quick_push (depend);
 if (start_ix == BUILT_IN_GOMP_TARGET)
 	args.quick_push (get_target_arguments (&gsi, entry_stmt));
 break;
 case BUILT_IN_GOACC_PARALLEL:
-oacc_set_fn_attrib (child_fn, clauses, &args);
+if (lookup_attribute ("oacc serial", DECL_ATTRIBUTES (child_fn)) != NULL)
+	{
+	  tree dims = NULL_TREE;
+	  unsigned int ix;
+	  /* For serial constructs we set all dimensions to 1.  */
+	  for (ix = GOMP_DIM_MAX; ix--;)
+	    dims = tree_cons (NULL_TREE, integer_one_node, dims);
+	  oacc_replace_fn_attrib (child_fn, dims);
+	}
+else
+	oacc_set_fn_attrib (child_fn, clauses, &args);
 tagging = true;
 /* FALLTHRU */
 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
 case BUILT_IN_GOACC_UPDATE:
 {
 }
 if (gpukernel->inner)
 expand_omp (gpukernel->inner);
 tree kern_fndecl = copy_node (orig_child_fndecl);
-DECL_NAME (kern_fndecl) = clone_function_name (kern_fndecl, "kernel");
+DECL_NAME (kern_fndecl) = clone_function_name_numbered (kern_fndecl,
+							  "kernel");
 SET_DECL_ASSEMBLER_NAME (kern_fndecl, DECL_NAME (kern_fndecl));
 tree tgtblock = gimple_block (tgt_stmt);
 tree fniniblock = make_node (BLOCK);
 BLOCK_ABSTRACT_ORIGIN (fniniblock) = BLOCK_ORIGIN (tgtblock);
 BLOCK_SOURCE_LOCATION (fniniblock) = BLOCK_SOURCE_LOCATION (tgtblock);
 DECL_INITIAL (kern_fndecl) = fniniblock;
 push_struct_function (kern_fndecl);
 cfun->function_end_locus = gimple_location (tgt_stmt);
 init_tree_ssa (cfun);
 pop_cfun ();
-/* Make sure to generate early debug for the function before
-outlining anything.  */
-if (! gimple_in_ssa_p (cfun))
-(*debug_hooks->early_global_decl) (cfun->decl);
 tree old_parm_decl = DECL_ARGUMENTS (kern_fndecl);
 gcc_assert (!DECL_CHAIN (old_parm_decl));
 tree new_parm_decl = copy_node (DECL_ARGUMENTS (kern_fndecl));
 DECL_CONTEXT (new_parm_decl) = kern_fndecl;
 		{
 		case GF_OMP_TARGET_KIND_REGION:
 		case GF_OMP_TARGET_KIND_DATA:
 		case GF_OMP_TARGET_KIND_OACC_PARALLEL:
 		case GF_OMP_TARGET_KIND_OACC_KERNELS:
+		case GF_OMP_TARGET_KIND_OACC_SERIAL:
 		case GF_OMP_TARGET_KIND_OACC_DATA:
 		case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
 		  break;
 		case GF_OMP_TARGET_KIND_UPDATE:
 		case GF_OMP_TARGET_KIND_ENTER_DATA:
 					 (as_a <gomp_ordered *> (stmt)),
 				       OMP_CLAUSE_DEPEND))
 	    /* #pragma omp ordered depend is also just a stand-alone
 	       directive.  */
 	    region = NULL;
+	  else if (code == GIMPLE_OMP_TASK
+		   && gimple_omp_task_taskwait_p (stmt))
+	    /* #pragma omp taskwait depend(...) is a stand-alone directive.  */
+	    region = NULL;
 	  /* ..., this directive becomes the parent for a new region.  */
 	  if (region)
 	    parent = region;
 	}
 }
 bool fallthru = false;
 switch (code)
 {
 case GIMPLE_OMP_PARALLEL:
-case GIMPLE_OMP_TASK:
 case GIMPLE_OMP_FOR:
 case GIMPLE_OMP_SINGLE:
 case GIMPLE_OMP_TEAMS:
 case GIMPLE_OMP_MASTER:
 case GIMPLE_OMP_TASKGROUP:
 case GIMPLE_OMP_GRID_BODY:
 cur_region = new_omp_region (bb, code, cur_region);
 fallthru = true;
 break;
+case GIMPLE_OMP_TASK:
+cur_region = new_omp_region (bb, code, cur_region);
+fallthru = true;
+if (gimple_omp_task_taskwait_p (last))
+	cur_region = cur_region->outer;
+break;
 case GIMPLE_OMP_ORDERED:
 cur_region = new_omp_region (bb, code, cur_region);
 fallthru = true;
 if (omp_find_clause (gimple_omp_ordered_clauses
 			     (as_a <gomp_ordered *> (last)),
 	{
 	case GF_OMP_TARGET_KIND_REGION:
 	case GF_OMP_TARGET_KIND_DATA:
 	case GF_OMP_TARGET_KIND_OACC_PARALLEL:
 	case GF_OMP_TARGET_KIND_OACC_KERNELS:
+	case GF_OMP_TARGET_KIND_OACC_SERIAL:
 	case GF_OMP_TARGET_KIND_OACC_DATA:
 	case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
 	  break;
 	case GF_OMP_TARGET_KIND_UPDATE:
 	case GF_OMP_TARGET_KIND_ENTER_DATA:

Mercurial > hg > CbC > CbC_gcc

comparison gcc/omp-expand.c @ 146:351920fa3827