CbC/CbC_gcc: gcc/tree-vect-data-refs.c comparison

comparison gcc/tree-vect-data-refs.c @ 145:1830386684a0

gcc-9.2.0

author	anatofuz
date	Thu, 13 Feb 2020 11:34:05 +0900
parents	84e7813d76e9
children

comparison

equal deleted inserted replaced

-:84e7813d76e9
+:1830386684a0
 /* Data References Analysis and Manipulation Utilities for Vectorization.
-Copyright (C) 2003-2018 Free Software Foundation, Inc.
+Copyright (C) 2003-2020 Free Software Foundation, Inc.
 Contributed by Dorit Naishlos <dorit@il.ibm.com>
 and Ira Rosen <irar@il.ibm.com>
 This file is part of GCC.
 #include "cfgloop.h"
 #include "tree-scalar-evolution.h"
 #include "tree-vectorizer.h"
 #include "expr.h"
 #include "builtins.h"
-#include "params.h"
 #include "tree-cfg.h"
 #include "tree-hash-traits.h"
 #include "vec-perm-indices.h"
 #include "internal-fn.h"
 rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
 if (rhs < lhs)
 scalar_type = rhs_type;
 }
+else if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
+{
+unsigned int i = 0;
+if (gimple_call_internal_p (call))
+	{
+	  internal_fn ifn = gimple_call_internal_fn (call);
+	  if (internal_load_fn_p (ifn) || internal_store_fn_p (ifn))
+	    /* gimple_expr_type already picked the type of the loaded
+	       or stored data.  */
+	    i = ~0U;
+	  else if (internal_fn_mask_index (ifn) == 0)
+	    i = 1;
+	}
+if (i < gimple_call_num_args (call))
+	{
+	  tree rhs_type = TREE_TYPE (gimple_call_arg (call, i));
+	  if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (rhs_type)))
+	    {
+	      rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
+	      if (rhs < lhs)
+		scalar_type = rhs_type;
+	    }
+	}
+}
 *lhs_size_unit = lhs;
 *rhs_size_unit = rhs;
 return scalar_type;
 }
 Return false if versioning is not supported.  */
 static opt_result
 vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
 {
-struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-if ((unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS) == 0)
+if ((unsigned) param_vect_max_version_for_alias_checks == 0)
 return opt_result::failure_at (vect_location,
 				   "will not create alias checks, as"
 				   " --param vect-max-version-for-alias-checks"
 				   " == 0\n");
 if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
 && !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
 return true;
 /* STMT_A and STMT_B belong to overlapping groups.  All loads in a
-group are emitted at the position of the last scalar load and all
+SLP group are emitted at the position of the last scalar load and
-stores in a group are emitted at the position of the last scalar store.
+all loads in an interleaving group are emitted at the position
+of the first scalar load.
+Stores in a group are emitted at the position of the last scalar store.
 Compute that position and check whether the resulting order matches
-the current one.  */
+the current one.
-stmt_vec_info last_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
+We have not yet decided between SLP and interleaving so we have
+to conservatively assume both.  */
+stmt_vec_info il_a;
+stmt_vec_info last_a = il_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
 if (last_a)
-for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_a); s;
+{
-	 s = DR_GROUP_NEXT_ELEMENT (s))
+for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_a); s;
-last_a = get_later_stmt (last_a, s);
+	   s = DR_GROUP_NEXT_ELEMENT (s))
+	last_a = get_later_stmt (last_a, s);
+if (!DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_a)))
+	{
+	  for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
+	       s = DR_GROUP_NEXT_ELEMENT (s))
+	    if (get_later_stmt (il_a, s) == il_a)
+	      il_a = s;
+	}
+else
+	il_a = last_a;
+}
 else
-last_a = stmtinfo_a;
+last_a = il_a = stmtinfo_a;
-stmt_vec_info last_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
+stmt_vec_info il_b;
+stmt_vec_info last_b = il_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
 if (last_b)
-for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_b); s;
+{
-	 s = DR_GROUP_NEXT_ELEMENT (s))
+for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_b); s;
-last_b = get_later_stmt (last_b, s);
+	   s = DR_GROUP_NEXT_ELEMENT (s))
+	last_b = get_later_stmt (last_b, s);
+if (!DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_b)))
+	{
+	  for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
+	       s = DR_GROUP_NEXT_ELEMENT (s))
+	    if (get_later_stmt (il_b, s) == il_b)
+	      il_b = s;
+	}
+else
+	il_b = last_b;
+}
 else
-last_b = stmtinfo_b;
+last_b = il_b = stmtinfo_b;
-return ((get_later_stmt (last_a, last_b) == last_a)
+bool a_after_b = (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a);
-	  == (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a));
+return (/* SLP */
+	  (get_later_stmt (last_a, last_b) == last_a) == a_after_b
+	  /* Interleaving */
+	  && (get_later_stmt (il_a, il_b) == il_a) == a_after_b
+	  /* Mixed */
+	  && (get_later_stmt (il_a, last_b) == il_a) == a_after_b
+	  && (get_later_stmt (last_a, il_b) == last_a) == a_after_b);
 }
 /* A subroutine of vect_analyze_data_ref_dependence.  Handle
 DDR_COULD_BE_INDEPENDENT_P ddr DDR that has a known set of dependence
 distances.  These distances are conservatively correct but they don't
 static bool
 vect_analyze_possibly_independent_ddr (data_dependence_relation *ddr,
 				       loop_vec_info loop_vinfo,
 				       int loop_depth, unsigned int *max_vf)
 {
-struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
 lambda_vector dist_v;
 unsigned int i;
 FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
 {
 int dist = dist_v[loop_depth];
 vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
 				  loop_vec_info loop_vinfo,
 				  unsigned int *max_vf)
 {
 unsigned int i;
-struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
 struct data_reference *dra = DDR_A (ddr);
 struct data_reference *drb = DDR_B (ddr);
 dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (dra);
 dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (drb);
 stmt_vec_info stmtinfo_a = dr_info_a->stmt;
 	{
 	  /* If DDR_REVERSED_P the order of the data-refs in DDR was
 	     reversed (to make distance vector positive), and the actual
 	     distance is negative.  */
 	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+	    dump_printf_loc (MSG_NOTE, vect_location,
 	                     "dependence distance negative.\n");
+	  /* When doing outer loop vectorization, we need to check if there is
+	     a backward dependence at the inner loop level if the dependence
+	     at the outer loop is reversed.  See PR81740.  */
+	  if (nested_in_vect_loop_p (loop, stmtinfo_a)
+	      || nested_in_vect_loop_p (loop, stmtinfo_b))
+	    {
+	      unsigned inner_depth = index_in_loop_nest (loop->inner->num,
+							 DDR_LOOP_NEST (ddr));
+	      if (dist_v[inner_depth] < 0)
+		return opt_result::failure_at (stmtinfo_a->stmt,
+					       "not vectorized, dependence "
+					       "between data-refs %T and %T\n",
+					       DR_REF (dra), DR_REF (drb));
+	    }
 	  /* Record a negative dependence distance to later limit the
 	     amount of stmt copying / unrolling we can perform.
 	     Only need to handle read-after-write dependence.  */
 	  if (DR_IS_READ (drb)
 	      && (STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) == 0
 unsigned int abs_dist = abs (dist);
 if (abs_dist >= 2 && abs_dist < *max_vf)
 	{
 	  /* The dependence distance requires reduction of the maximal
 	     vectorization factor.  */
-	  *max_vf = abs (dist);
+	  *max_vf = abs_dist;
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 	                     "adjusting maximal vectorization factor to %i\n",
 	                     *max_vf);
 	}
 void
 vect_record_base_alignments (vec_info *vinfo)
 {
 loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
+class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
 data_reference *dr;
 unsigned int i;
 FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
 {
 dr_vec_info *dr_info = vinfo->lookup_dr (dr);
 }
 }
 /* Return the target alignment for the vectorized form of DR_INFO.  */
-static unsigned int
+static poly_uint64
 vect_calculate_target_alignment (dr_vec_info *dr_info)
 {
 tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
 return targetm.vectorize.preferred_vector_alignment (vectype);
 }
 vect_compute_data_ref_alignment (dr_vec_info *dr_info)
 {
 stmt_vec_info stmt_info = dr_info->stmt;
 vec_base_alignments *base_alignments = &stmt_info->vinfo->base_alignments;
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
-struct loop *loop = NULL;
+class loop *loop = NULL;
 tree ref = DR_REF (dr_info->dr);
 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 if (dump_enabled_p ())
 dump_printf_loc (MSG_NOTE, vect_location,
 return;
 innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
 bool step_preserves_misalignment_p;
-unsigned HOST_WIDE_INT vector_alignment
+poly_uint64 vector_alignment
-= vect_calculate_target_alignment (dr_info) / BITS_PER_UNIT;
+= exact_div (vect_calculate_target_alignment (dr_info), BITS_PER_UNIT);
 DR_TARGET_ALIGNMENT (dr_info) = vector_alignment;
+/* If the main loop has peeled for alignment we have no way of knowing
+whether the data accesses in the epilogues are aligned.  We can't at
+compile time answer the question whether we have entered the main loop or
+not.  Fixes PR 92351.  */
+if (loop_vinfo)
+{
+loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
+if (orig_loop_vinfo
+	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo) != 0)
+	return;
+}
+unsigned HOST_WIDE_INT vect_align_c;
+if (!vector_alignment.is_constant (&vect_align_c))
+return;
 /* No step for BB vectorization.  */
 if (!loop)
 {
 gcc_assert (integer_zerop (drb->step));
 we have to check that the stride of the dataref in the inner-loop evenly
 divides by the vector alignment.  */
 else if (nested_in_vect_loop_p (loop, stmt_info))
 {
 step_preserves_misalignment_p
-	= (DR_STEP_ALIGNMENT (dr_info->dr) % vector_alignment) == 0;
+	= (DR_STEP_ALIGNMENT (dr_info->dr) % vect_align_c) == 0;
 if (dump_enabled_p ())
 	{
 	  if (step_preserves_misalignment_p)
 	    dump_printf_loc (MSG_NOTE, vect_location,
 the dataref evenly divides by the alignment.  */
 else
 {
 poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
 step_preserves_misalignment_p
-	= multiple_p (DR_STEP_ALIGNMENT (dr_info->dr) * vf, vector_alignment);
+	= multiple_p (DR_STEP_ALIGNMENT (dr_info->dr) * vf, vect_align_c);
 if (!step_preserves_misalignment_p && dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "step doesn't divide the vector alignment.\n");
 }
 {
 base_alignment = (*entry)->base_alignment;
 base_misalignment = (*entry)->base_misalignment;
 }
-if (drb->offset_alignment < vector_alignment
+if (drb->offset_alignment < vect_align_c
 || !step_preserves_misalignment_p
 /* We need to know whether the step wrt the vectorized loop is
 	 negative when computing the starting misalignment below.  */
 || TREE_CODE (drb->step) != INTEGER_CST)
 {
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "Unknown alignment for access: %T\n", ref);
 return;
 }
-if (base_alignment < vector_alignment)
+if (base_alignment < vect_align_c)
 {
 unsigned int max_alignment;
 tree base = get_base_for_alignment (drb->base_address, &max_alignment);
-if (max_alignment < vector_alignment
+if (max_alignment < vect_align_c
 	  || !vect_can_force_dr_alignment_p (base,
-					     vector_alignment * BITS_PER_UNIT))
+					     vect_align_c * BITS_PER_UNIT))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "can't force alignment of ref: %T\n", ref);
 	  return;
 /* PLUS because STEP is negative.  */
 misalignment += ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
 		     * TREE_INT_CST_LOW (drb->step));
 unsigned int const_misalignment;
-if (!known_misalignment (misalignment, vector_alignment,
+if (!known_misalignment (misalignment, vect_align_c, &const_misalignment))
-			   &const_misalignment))
 {
 if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "Non-constant misalignment for access: %T\n", ref);
 return;
 				   dr_vec_info *dr_peel_info, int npeel)
 {
 unsigned int i;
 vec<dr_p> same_aligned_drs;
 struct data_reference *current_dr;
-int dr_size = vect_get_scalar_dr_size (dr_info);
-int dr_peel_size = vect_get_scalar_dr_size (dr_peel_info);
-stmt_vec_info stmt_info = dr_info->stmt;
 stmt_vec_info peel_stmt_info = dr_peel_info->stmt;
-/* For interleaved data accesses the step in the loop must be multiplied by
+/* It can be assumed that if dr_info has the same alignment as dr_peel,
-the size of the interleaving group.  */
+it is aligned in the vector loop.  */
-if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
-dr_size *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
-if (STMT_VINFO_GROUPED_ACCESS (peel_stmt_info))
-dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info);
-/* It can be assumed that the data refs with the same alignment as dr_peel
-are aligned in the vector loop.  */
 same_aligned_drs = STMT_VINFO_SAME_ALIGN_REFS (peel_stmt_info);
 FOR_EACH_VEC_ELT (same_aligned_drs, i, current_dr)
 {
 if (current_dr != dr_info->dr)
 continue;
 gcc_assert (!known_alignment_for_access_p (dr_info)
 		  || !known_alignment_for_access_p (dr_peel_info)
-		  || (DR_MISALIGNMENT (dr_info) / dr_size
+		  || (DR_MISALIGNMENT (dr_info)
-		      == DR_MISALIGNMENT (dr_peel_info) / dr_peel_size));
+		      == DR_MISALIGNMENT (dr_peel_info)));
 SET_DR_MISALIGNMENT (dr_info, 0);
 return;
 }
-if (known_alignment_for_access_p (dr_info)
+unsigned HOST_WIDE_INT alignment;
+if (DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)
+&& known_alignment_for_access_p (dr_info)
 && known_alignment_for_access_p (dr_peel_info))
 {
-bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
-					    size_zero_node) < 0;
 int misal = DR_MISALIGNMENT (dr_info);
-misal += negative ? -npeel * dr_size : npeel * dr_size;
+misal += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
-misal &= DR_TARGET_ALIGNMENT (dr_info) - 1;
+misal &= alignment - 1;
 SET_DR_MISALIGNMENT (dr_info, misal);
 return;
 }
 if (dump_enabled_p ())
 opt_result
 vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 {
 vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
-struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
 enum dr_alignment_support supportable_dr_alignment;
 dr_vec_info *first_store = NULL;
 dr_vec_info *dr0_info = NULL;
 struct data_reference *dr;
 unsigned int i, j;
 	      unsigned int npeel_tmp = 0;
 	      bool negative = tree_int_cst_compare (DR_STEP (dr),
 						    size_zero_node) < 0;
 	      vectype = STMT_VINFO_VECTYPE (stmt_info);
-	      unsigned int target_align = DR_TARGET_ALIGNMENT (dr_info);
+	      /* If known_alignment_for_access_p then we have set
+	         DR_MISALIGNMENT which is only done if we know it at compiler
+	         time, so it is safe to assume target alignment is constant.
+	       */
+	      unsigned int target_align =
+		DR_TARGET_ALIGNMENT (dr_info).to_constant ();
 	      unsigned int dr_size = vect_get_scalar_dr_size (dr_info);
 	      mis = (negative
 		     ? DR_MISALIGNMENT (dr_info)
 		     : -DR_MISALIGNMENT (dr_info));
 	      if (DR_MISALIGNMENT (dr_info) != 0)
 	      /* For data-refs with the same number of related
 		 accesses prefer the one where the misalign
 		 computation will be invariant in the outermost loop.  */
 	      else if (same_align_drs_max == same_align_drs)
 		{
-		  struct loop *ivloop0, *ivloop;
+		  class loop *ivloop0, *ivloop;
 		  ivloop0 = outermost_invariant_loop_for_expr
 		    (loop, DR_BASE_ADDRESS (dr0_info->dr));
 		  ivloop = outermost_invariant_loop_for_expr
 		    (loop, DR_BASE_ADDRESS (dr));
 		  if ((ivloop && !ivloop0)
 vectorization factor minus the misalignment as an element
 count.  */
 	      mis = (negative
 		     ? DR_MISALIGNMENT (dr0_info)
 		     : -DR_MISALIGNMENT (dr0_info));
-	      unsigned int target_align = DR_TARGET_ALIGNMENT (dr0_info);
+	      /* If known_alignment_for_access_p then we have set
+	         DR_MISALIGNMENT which is only done if we know it at compiler
+	         time, so it is safe to assume target alignment is constant.
+	       */
+	      unsigned int target_align =
+		DR_TARGET_ALIGNMENT (dr0_info).to_constant ();
 	      npeel = ((mis & (target_align - 1))
 		       / vect_get_scalar_dr_size (dr0_info));
 }
 	  /* For interleaved data access every iteration accesses all the
 /* Cost model #1 - honor --param vect-max-peeling-for-alignment.  */
 if (do_peeling)
 {
 unsigned max_allowed_peel
-= PARAM_VALUE (PARAM_VECT_MAX_PEELING_FOR_ALIGNMENT);
+	    = param_vect_max_peeling_for_alignment;
+	  if (flag_vect_cost_model == VECT_COST_MODEL_CHEAP)
+	    max_allowed_peel = 0;
 if (max_allowed_peel != (unsigned)-1)
 {
 unsigned max_peel = npeel;
 if (max_peel == 0)
 {
-		  unsigned int target_align = DR_TARGET_ALIGNMENT (dr0_info);
+		  poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr0_info);
-		  max_peel = (target_align
+		  unsigned HOST_WIDE_INT target_align_c;
-			      / vect_get_scalar_dr_size (dr0_info) - 1);
+		  if (target_align.is_constant (&target_align_c))
+		    max_peel =
+		      target_align_c / vect_get_scalar_dr_size (dr0_info) - 1;
+		  else
+		    {
+		      do_peeling = false;
+		      if (dump_enabled_p ())
+			dump_printf_loc (MSG_NOTE, vect_location,
+			  "Disable peeling, max peels set and vector"
+			  " alignment unknown\n");
+		    }
 }
 if (max_peel > max_allowed_peel)
 {
 do_peeling = false;
 if (dump_enabled_p ())
 }
 /* (2) Versioning to force alignment.  */
 /* Try versioning if:
-1) optimize loop for speed
+1) optimize loop for speed and the cost-model is not cheap
 2) there is at least one unsupported misaligned data ref with an unknown
 misalignment, and
 3) all misaligned data refs with a known misalignment are supported, and
 4) the number of runtime alignment checks is within reason.  */
-do_versioning =
+do_versioning
-	optimize_loop_nest_for_speed_p (loop)
+= (optimize_loop_nest_for_speed_p (loop)
-	&& (!loop->inner); /* FORNOW */
+&& !loop->inner /* FORNOW */
+&& flag_vect_cost_model != VECT_COST_MODEL_CHEAP);
 if (do_versioning)
 {
 FOR_EACH_VEC_ELT (datarefs, i, dr)
 {
 int mask;
 tree vectype;
 if (known_alignment_for_access_p (dr_info)
 || LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
->= (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS))
+		  >= (unsigned) param_vect_max_version_for_alignment_checks)
 {
 do_versioning = false;
 break;
 }
 		{
 		  do_versioning = false;
 		  break;
 		}
+	      /* Forcing alignment in the first iteration is no good if
+		 we don't keep it across iterations.  For now, just disable
+		 versioning in this case.
+		 ?? We could actually unroll the loop to achieve the required
+		 overall step alignment, and forcing the alignment could be
+		 done by doing some iterations of the non-vectorized loop.  */
+	      if (!multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+			       * DR_STEP_ALIGNMENT (dr),
+			       DR_TARGET_ALIGNMENT (dr_info)))
+		{
+		  do_versioning = false;
+		  break;
+		}
 /* The rightmost bits of an aligned address must be zeros.
 Construct the mask needed for this test.  For example,
 GET_MODE_SIZE for the vector mode V4SI is 16 bytes so the
 mask must be 15 = 0xf. */
 	      mask = size - 1;
-/* FORNOW: use the same mask to test all potentially unaligned
+	      /* FORNOW: use the same mask to test all potentially unaligned
-references in the loop.  The vectorizer currently supports
+		 references in the loop.  */
-a single vector size, see the reference to
+	      if (LOOP_VINFO_PTR_MASK (loop_vinfo)
-GET_MODE_NUNITS (TYPE_MODE (vectype)) where the
+		  && LOOP_VINFO_PTR_MASK (loop_vinfo) != mask)
-vectorization factor is computed.  */
+		{
-gcc_assert (!LOOP_VINFO_PTR_MASK (loop_vinfo)
+		  do_versioning = false;
-|| LOOP_VINFO_PTR_MASK (loop_vinfo) == mask);
+		  break;
+		}
 LOOP_VINFO_PTR_MASK (loop_vinfo) = mask;
 	      LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).safe_push (stmt_info);
 }
 }
 poly_offset_int diff = (wi::to_poly_offset (DR_INIT (dra))
 			  - wi::to_poly_offset (DR_INIT (drb)));
 if (maybe_ne (diff, 0))
 {
 /* Get the wider of the two alignments.  */
-unsigned int align_a = (vect_calculate_target_alignment (dr_info_a)
+poly_uint64 align_a =
-			      / BITS_PER_UNIT);
+	exact_div (vect_calculate_target_alignment (dr_info_a),
-unsigned int align_b = (vect_calculate_target_alignment (dr_info_b)
+		   BITS_PER_UNIT);
-			      / BITS_PER_UNIT);
+poly_uint64 align_b =
-unsigned int max_align = MAX (align_a, align_b);
+	exact_div (vect_calculate_target_alignment (dr_info_b),
+		   BITS_PER_UNIT);
+unsigned HOST_WIDE_INT align_a_c, align_b_c;
+if (!align_a.is_constant (&align_a_c)
+	  || !align_b.is_constant (&align_b_c))
+	return;
+unsigned HOST_WIDE_INT max_align = MAX (align_a_c, align_b_c);
 /* Require the gap to be a multiple of the larger vector alignment.  */
 if (!multiple_p (diff, max_align))
 	return;
 }
 	  /* Mark the statement as unvectorizable.  */
 	  STMT_VINFO_VECTORIZABLE (stmt_info) = false;
 	  return true;
 	}
-dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
+if (dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
 STMT_VINFO_STRIDED_P (stmt_info) = true;
 return true;
 }
 if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
 /* First stmt in the interleaving chain. Check the chain.  */
 stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
 struct data_reference *data_ref = dr;
 unsigned int count = 1;
 tree prev_init = DR_INIT (data_ref);
-stmt_vec_info prev = stmt_info;
 HOST_WIDE_INT diff, gaps = 0;
 /* By construction, all group members have INTEGER_CST DR_INITs.  */
 while (next)
 {
-/* Skip same data-refs.  In case that two or more stmts share
+/* We never have the same DR multiple times.  */
-data-ref (supported only for loads), we vectorize only the first
+gcc_assert (tree_int_cst_compare (DR_INIT (data_ref),
-stmt, and the rest get their vectorized loads from the first
+				DR_INIT (STMT_VINFO_DATA_REF (next))) != 0);
-one.  */
-if (!tree_int_cst_compare (DR_INIT (data_ref),
-				     DR_INIT (STMT_VINFO_DATA_REF (next))))
-{
-if (DR_IS_WRITE (data_ref))
-{
-if (dump_enabled_p ())
-dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-"Two store stmts share the same dr.\n");
-return false;
-}
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "Two or more load stmts share the same dr.\n");
-	      /* For load use the same data-ref load.  */
-	      DR_GROUP_SAME_DR_STMT (next) = prev;
-	      prev = next;
-	      next = DR_GROUP_NEXT_ELEMENT (next);
-	      continue;
-}
-	  prev = next;
 	  data_ref = STMT_VINFO_DATA_REF (next);
 	  /* All group members have the same STEP by construction.  */
 	  gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0));
 	    dump_printf (MSG_NOTE, "load ");
 	  else if (STMT_VINFO_STRIDED_P (stmt_info))
 	    dump_printf (MSG_NOTE, "strided store ");
 	  else
 	    dump_printf (MSG_NOTE, "store ");
-	  dump_printf (MSG_NOTE, "of size %u starting with %G",
+	  dump_printf (MSG_NOTE, "of size %u\n",
-		       (unsigned)groupsize, stmt_info->stmt);
+		       (unsigned)groupsize);
+	  dump_printf_loc (MSG_NOTE, vect_location, "\t%G", stmt_info->stmt);
+	  next = DR_GROUP_NEXT_ELEMENT (stmt_info);
+	  while (next)
+	    {
+	      if (DR_GROUP_GAP (next) != 1)
+		dump_printf_loc (MSG_NOTE, vect_location,
+				 "\t<gap of %d elements>\n",
+				 DR_GROUP_GAP (next) - 1);
+	      dump_printf_loc (MSG_NOTE, vect_location, "\t%G", next->stmt);
+	      next = DR_GROUP_NEXT_ELEMENT (next);
+	    }
 	  if (DR_GROUP_GAP (stmt_info) != 0)
 	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "There is a gap of %u elements after the group\n",
+			     "\t<gap of %d elements>\n",
 			     DR_GROUP_GAP (stmt_info));
 	}
 /* SLP: create an SLP data structure for every interleaving group of
 	 stores for further analysis in vect_analyse_slp.  */
 data_reference *dr = dr_info->dr;
 tree step = DR_STEP (dr);
 tree scalar_type = TREE_TYPE (DR_REF (dr));
 stmt_vec_info stmt_info = dr_info->stmt;
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
-struct loop *loop = NULL;
+class loop *loop = NULL;
 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
 return true;
 if (loop_vinfo)
 return NULL_TREE;
 return gimple_assign_rhs1 (stmt);
 }
 /* Return true if vectorizable_* routines can handle statements STMT1_INFO
-and STMT2_INFO being in a single group.  */
+and STMT2_INFO being in a single group.  When ALLOW_SLP_P, masked loads can
+be grouped in SLP mode.  */
 static bool
-can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
+can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info,
+		   bool allow_slp_p)
 {
 if (gimple_assign_single_p (stmt1_info->stmt))
 return gimple_assign_single_p (stmt2_info->stmt);
 gcall *call1 = dyn_cast <gcall *> (stmt1_info->stmt);
 /* Check that the masks are the same.  Cope with casts of masks,
 	 like those created by build_mask_conversion.  */
 tree mask1 = gimple_call_arg (call1, 2);
 tree mask2 = gimple_call_arg (call2, 2);
-if (!operand_equal_p (mask1, mask2, 0))
+if (!operand_equal_p (mask1, mask2, 0)
+&& (ifn == IFN_MASK_STORE || !allow_slp_p))
 	{
 	  mask1 = strip_conversion (mask1);
 	  if (!mask1)
 	    return false;
 	  mask2 = strip_conversion (mask2);
 	     not masked loads or stores).  */
 	  if (DR_IS_READ (dra) != DR_IS_READ (drb)
 	      || data_ref_compare_tree (DR_BASE_ADDRESS (dra),
 					DR_BASE_ADDRESS (drb)) != 0
 	      || data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)) != 0
-	      || !can_group_stmts_p (stmtinfo_a, stmtinfo_b))
+	      || !can_group_stmts_p (stmtinfo_a, stmtinfo_b, true))
 	    break;
 	  /* Check that the data-refs have the same constant size.  */
 	  tree sza = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra)));
 	  tree szb = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb)));
 	    break;
 	  /* Check that the DR_INITs are compile-time constants.  */
 	  if (TREE_CODE (DR_INIT (dra)) != INTEGER_CST
 	      || TREE_CODE (DR_INIT (drb)) != INTEGER_CST)
+	    break;
+	  /* Different .GOMP_SIMD_LANE calls still give the same lane,
+	     just hold extra information.  */
+	  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
+	      && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
+	      && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
 	    break;
 	  /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb).  */
 	  HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
 	  HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
 	    }
 	  DR_GROUP_FIRST_ELEMENT (stmtinfo_b) = stmtinfo_a;
 	  DR_GROUP_NEXT_ELEMENT (lastinfo) = stmtinfo_b;
 	  lastinfo = stmtinfo_b;
+	  STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a)
+	    = !can_group_stmts_p (stmtinfo_a, stmtinfo_b, false);
+	  if (dump_enabled_p () && STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
+	    dump_printf_loc (MSG_NOTE, vect_location,
+			     "Load suitable for SLP vectorization only.\n");
 	  if (init_b == init_prev
 	      && !to_fixup.add (DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
 	      && dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "Queuing group with duplicate access for fixup\n");
 /* Find the earliest duplicate group member.  */
 unsigned first_duplicate = -1u;
 stmt_vec_info next, g = grp;
 while ((next = DR_GROUP_NEXT_ELEMENT (g)))
 	{
-	  if ((DR_INIT (STMT_VINFO_DR_INFO (next)->dr)
+	  if (tree_int_cst_equal (DR_INIT (STMT_VINFO_DR_INFO (next)->dr),
-	       == DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
+				  DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
 	      && gimple_uid (STMT_VINFO_STMT (next)) < first_duplicate)
 	    first_duplicate = gimple_uid (STMT_VINFO_STMT (next));
 	  g = next;
 	}
 if (first_duplicate == -1U)
 /* Then move all stmts after the first duplicate to a new group.
 Note this is a heuristic but one with the property that *it
 	 is fixed up completely.  */
 g = grp;
-stmt_vec_info newgroup = NULL, ng;
+stmt_vec_info newgroup = NULL, ng = grp;
 while ((next = DR_GROUP_NEXT_ELEMENT (g)))
 	{
 	  if (gimple_uid (STMT_VINFO_STMT (next)) >= first_duplicate)
 	    {
 	      DR_GROUP_NEXT_ELEMENT (g) = DR_GROUP_NEXT_ELEMENT (next);
 			  LOOP_VINFO_LOOP_NEST (loop_vinfo));
 /* First, we collect all data ref pairs for aliasing checks.  */
 FOR_EACH_VEC_ELT (may_alias_ddrs, i, ddr)
 {
-int comp_res;
 poly_uint64 lower_bound;
 tree segment_length_a, segment_length_b;
 unsigned HOST_WIDE_INT access_size_a, access_size_b;
 unsigned int align_a, align_b;
 stmt_vec_info stmt_info_a = dr_info_a->stmt;
 dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
 stmt_vec_info stmt_info_b = dr_info_b->stmt;
+bool preserves_scalar_order_p
+	= vect_preserves_scalar_order_p (dr_info_a, dr_info_b);
 /* Skip the pair if inter-iteration dependencies are irrelevant
 	 and intra-iteration dependencies are guaranteed to be honored.  */
 if (ignore_step_p
-	  && (vect_preserves_scalar_order_p (dr_info_a, dr_info_b)
+	  && (preserves_scalar_order_p
 	      || vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
 						 &lower_bound)))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 access_size_a = vect_vfa_access_size (dr_info_a);
 access_size_b = vect_vfa_access_size (dr_info_b);
 align_a = vect_vfa_align (dr_info_a);
 align_b = vect_vfa_align (dr_info_b);
-comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (dr_info_a->dr),
-					DR_BASE_ADDRESS (dr_info_b->dr));
-if (comp_res == 0)
-	comp_res = data_ref_compare_tree (DR_OFFSET (dr_info_a->dr),
-					  DR_OFFSET (dr_info_b->dr));
 /* See whether the alias is known at compilation time.  */
-if (comp_res == 0
+if (operand_equal_p (DR_BASE_ADDRESS (dr_info_a->dr),
+			   DR_BASE_ADDRESS (dr_info_b->dr), 0)
+	  && operand_equal_p (DR_OFFSET (dr_info_a->dr),
+			      DR_OFFSET (dr_info_b->dr), 0)
 	  && TREE_CODE (DR_STEP (dr_info_a->dr)) == INTEGER_CST
 	  && TREE_CODE (DR_STEP (dr_info_b->dr)) == INTEGER_CST
 	  && poly_int_tree_p (segment_length_a)
 	  && poly_int_tree_p (segment_length_b))
 	{
 					   " compilation time alias: %G%G",
 					   stmt_info_a->stmt,
 					   stmt_info_b->stmt);
 	}
+dr_with_seg_len dr_a (dr_info_a->dr, segment_length_a,
+			    access_size_a, align_a);
+dr_with_seg_len dr_b (dr_info_b->dr, segment_length_b,
+			    access_size_b, align_b);
+/* Canonicalize the order to be the one that's needed for accurate
+	 RAW, WAR and WAW flags, in cases where the data references are
+	 well-ordered.  The order doesn't really matter otherwise,
+	 but we might as well be consistent.  */
+if (get_later_stmt (stmt_info_a, stmt_info_b) == stmt_info_a)
+	std::swap (dr_a, dr_b);
 dr_with_seg_len_pair_t dr_with_seg_len_pair
-	(dr_with_seg_len (dr_info_a->dr, segment_length_a,
+	(dr_a, dr_b, (preserves_scalar_order_p
-			  access_size_a, align_a),
+		      ? dr_with_seg_len_pair_t::WELL_ORDERED
-	 dr_with_seg_len (dr_info_b->dr, segment_length_b,
+		      : dr_with_seg_len_pair_t::REORDERED));
-			  access_size_b, align_b));
-/* Canonicalize pairs by sorting the two DR members.  */
-if (comp_res > 0)
-	std::swap (dr_with_seg_len_pair.first, dr_with_seg_len_pair.second);
 comp_alias_ddrs.safe_push (dr_with_seg_len_pair);
 }
 prune_runtime_alias_test_list (&comp_alias_ddrs, vect_factor);
 unsigned int count = (comp_alias_ddrs.length ()
 			+ check_unequal_addrs.length ());
-dump_printf_loc (MSG_NOTE, vect_location,
+if (dump_enabled_p ())
-		   "improved number of alias checks from %d to %d\n",
+dump_printf_loc (MSG_NOTE, vect_location,
-		   may_alias_ddrs.length (), count);
+		     "improved number of alias checks from %d to %d\n",
-if ((int) count > PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS))
+		     may_alias_ddrs.length (), count);
+unsigned limit = param_vect_max_version_for_alias_checks;
+if (flag_simd_cost_model == VECT_COST_MODEL_CHEAP)
+limit = param_vect_max_version_for_alias_checks * 6 / 10;
+if (count > limit)
 return opt_result::failure_at
 (vect_location,
-"number of versioning for alias "
+"number of versioning for alias run-time tests exceeds %d "
-"run-time tests exceeds %d "
+"(--param vect-max-version-for-alias-checks)\n", limit);
-"(--param vect-max-version-for-alias-checks)\n",
-PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS));
 return opt_result::success ();
 }
 /* Check whether we can use an internal function for a gather load
 or scatter store.  READ_P is true for loads and false for stores.
 MASKED_P is true if the load or store is conditional.  MEMORY_TYPE is
-the type of the memory elements being loaded or stored.  OFFSET_BITS
+the type of the memory elements being loaded or stored.  OFFSET_TYPE
-is the number of bits in each scalar offset and OFFSET_SIGN is the
+is the type of the offset that is being applied to the invariant
-sign of the offset.  SCALE is the amount by which the offset should
+base address.  SCALE is the amount by which the offset should
 be multiplied *after* it has been converted to address width.
-Return true if the function is supported, storing the function
+Return true if the function is supported, storing the function id in
-id in *IFN_OUT and the type of a vector element in *ELEMENT_TYPE_OUT.  */
+*IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.  */
 bool
-vect_gather_scatter_fn_p (bool read_p, bool masked_p, tree vectype,
+vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
-			  tree memory_type, unsigned int offset_bits,
+			  tree vectype, tree memory_type, tree offset_type,
-			  signop offset_sign, int scale,
+			  int scale, internal_fn *ifn_out,
-			  internal_fn *ifn_out, tree *element_type_out)
+			  tree *offset_vectype_out)
 {
 unsigned int memory_bits = tree_to_uhwi (TYPE_SIZE (memory_type));
 unsigned int element_bits = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
-if (offset_bits > element_bits)
-/* Internal functions require the offset to be the same width as
-the vector elements.  We can extend narrower offsets, but it isn't
-safe to truncate wider offsets.  */
-return false;
 if (element_bits != memory_bits)
 /* For now the vector elements must be the same width as the
 memory elements.  */
 return false;
 if (read_p)
 ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
 else
 ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
-/* Test whether the target supports this combination.  */
+for (;;)
-if (!internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
+{
-					       offset_sign, scale))
+tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
-return false;
+if (!offset_vectype)
+	return false;
-*ifn_out = ifn;
-*element_type_out = TREE_TYPE (vectype);
+/* Test whether the target supports this combination.  */
-return true;
+if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
+						  offset_vectype, scale))
+	{
+	  *ifn_out = ifn;
+	  *offset_vectype_out = offset_vectype;
+	  return true;
+	}
+if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
+	  && TYPE_PRECISION (offset_type) >= element_bits)
+	return false;
+offset_type = build_nonstandard_integer_type
+	(TYPE_PRECISION (offset_type) * 2, TYPE_UNSIGNED (offset_type));
+}
 }
 /* STMT_INFO is a call to an internal gather load or scatter store function.
 Describe the operation in INFO.  */
 vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
 			   gather_scatter_info *info)
 {
 HOST_WIDE_INT scale = 1;
 poly_int64 pbitpos, pbitsize;
-struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
 tree offtype = NULL_TREE;
 tree decl = NULL_TREE, base, off;
 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 tree memory_type = TREE_TYPE (DR_REF (dr));
 machine_mode pmode;
 int punsignedp, reversep, pvolatilep = 0;
 internal_fn ifn;
-tree element_type;
+tree offset_vectype;
 bool masked_p = false;
 /* See whether this is already a call to a gather/scatter internal function.
 If not, see whether it's a masked load or store.  */
 gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
 	case MULT_EXPR:
 	  if (scale == 1 && tree_fits_shwi_p (op1))
 	    {
 	      int new_scale = tree_to_shwi (op1);
 	      /* Only treat this as a scaling operation if the target
-		 supports it.  */
+		 supports it for at least some offset type.  */
 	      if (use_ifn_p
-		  && !vect_gather_scatter_fn_p (DR_IS_READ (dr), masked_p,
+		  && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
-						vectype, memory_type, 1,
+						masked_p, vectype, memory_type,
-						TYPE_SIGN (TREE_TYPE (op0)),
+						signed_char_type_node,
 						new_scale, &ifn,
-						&element_type))
+						&offset_vectype)
+		  && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
+						masked_p, vectype, memory_type,
+						unsigned_char_type_node,
+						new_scale, &ifn,
+						&offset_vectype))
 		break;
 	      scale = new_scale;
 	      off = op0;
 	      continue;
 	    }
 	  continue;
 	CASE_CONVERT:
 	  if (!POINTER_TYPE_P (TREE_TYPE (op0))
 	      && !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
 	    break;
+	  /* Don't include the conversion if the target is happy with
+	     the current offset type.  */
+	  if (use_ifn_p
+	      && vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
+					   masked_p, vectype, memory_type,
+					   TREE_TYPE (off), scale, &ifn,
+					   &offset_vectype))
+	    break;
 	  if (TYPE_PRECISION (TREE_TYPE (op0))
 	      == TYPE_PRECISION (TREE_TYPE (off)))
 	    {
 	      off = op0;
 	      continue;
 	    }
-	  /* The internal functions need the offset to be the same width
-	     as the elements of VECTYPE.  Don't include operations that
-	     cast the offset from that width to a different width.  */
-	  if (use_ifn_p
-	      && (int_size_in_bytes (TREE_TYPE (vectype))
-		  == int_size_in_bytes (TREE_TYPE (off))))
-	    break;
 	  if (TYPE_PRECISION (TREE_TYPE (op0))
 	      < TYPE_PRECISION (TREE_TYPE (off)))
 	    {
 	      off = op0;
 if (offtype == NULL_TREE)
 offtype = TREE_TYPE (off);
 if (use_ifn_p)
 {
-if (!vect_gather_scatter_fn_p (DR_IS_READ (dr), masked_p, vectype,
+if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
-				     memory_type, TYPE_PRECISION (offtype),
+				     vectype, memory_type, offtype, scale,
-				     TYPE_SIGN (offtype), scale, &ifn,
+				     &ifn, &offset_vectype))
-				     &element_type))
 	return false;
 }
 else
 {
 if (DR_IS_READ (dr))
 if (!decl)
 	return false;
 ifn = IFN_LAST;
-element_type = TREE_TYPE (vectype);
+/* The offset vector type will be read from DECL when needed.  */
+offset_vectype = NULL_TREE;
 }
 info->ifn = ifn;
 info->decl = decl;
 info->base = base;
 info->offset = off;
 info->offset_dt = vect_unknown_def_type;
-info->offset_vectype = NULL_TREE;
+info->offset_vectype = offset_vectype;
 info->scale = scale;
-info->element_type = element_type;
+info->element_type = TREE_TYPE (vectype);
 info->memory_type = memory_type;
 return true;
 }
 /* Find the data references in STMT, analyze them with respect to LOOP and
 			   DR_IS_READ (dr), DR_IS_CONDITIONAL_IN_STMT (dr));
 if (DR_BASE_ADDRESS (newdr)
 	  && DR_OFFSET (newdr)
 	  && DR_INIT (newdr)
 	  && DR_STEP (newdr)
+	  && TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
 	  && integer_zerop (DR_STEP (newdr)))
 	{
+	  tree base_address = DR_BASE_ADDRESS (newdr);
 	  tree off = DR_OFFSET (newdr);
+	  tree step = ssize_int (1);
+	  if (integer_zerop (off)
+	      && TREE_CODE (base_address) == POINTER_PLUS_EXPR)
+	    {
+	      off = TREE_OPERAND (base_address, 1);
+	      base_address = TREE_OPERAND (base_address, 0);
+	    }
 	  STRIP_NOPS (off);
-	  if (TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
+	  if (TREE_CODE (off) == MULT_EXPR
-	      && TREE_CODE (off) == MULT_EXPR
 	      && tree_fits_uhwi_p (TREE_OPERAND (off, 1)))
 	    {
-	      tree step = TREE_OPERAND (off, 1);
+	      step = TREE_OPERAND (off, 1);
 	      off = TREE_OPERAND (off, 0);
 	      STRIP_NOPS (off);
-	      if (CONVERT_EXPR_P (off)
+	    }
-		  && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
+	  if (CONVERT_EXPR_P (off)
-		      < TYPE_PRECISION (TREE_TYPE (off))))
+	      && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
-		off = TREE_OPERAND (off, 0);
+		  < TYPE_PRECISION (TREE_TYPE (off))))
-	      if (TREE_CODE (off) == SSA_NAME)
+	    off = TREE_OPERAND (off, 0);
+	  if (TREE_CODE (off) == SSA_NAME)
+	    {
+	      gimple *def = SSA_NAME_DEF_STMT (off);
+	      /* Look through widening conversion.  */
+	      if (is_gimple_assign (def)
+		  && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
 		{
-		  gimple *def = SSA_NAME_DEF_STMT (off);
+		  tree rhs1 = gimple_assign_rhs1 (def);
+		  if (TREE_CODE (rhs1) == SSA_NAME
+		      && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
+		      && (TYPE_PRECISION (TREE_TYPE (off))
+			  > TYPE_PRECISION (TREE_TYPE (rhs1))))
+		    def = SSA_NAME_DEF_STMT (rhs1);
+		}
+	      if (is_gimple_call (def)
+		  && gimple_call_internal_p (def)
+		  && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
+		{
+		  tree arg = gimple_call_arg (def, 0);
 		  tree reft = TREE_TYPE (DR_REF (newdr));
-		  if (is_gimple_call (def)
+		  gcc_assert (TREE_CODE (arg) == SSA_NAME);
-		      && gimple_call_internal_p (def)
+		  arg = SSA_NAME_VAR (arg);
-		      && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
+		  if (arg == loop->simduid
+		      /* For now.  */
+		      && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
 		    {
-		      tree arg = gimple_call_arg (def, 0);
+		      DR_BASE_ADDRESS (newdr) = base_address;
-		      gcc_assert (TREE_CODE (arg) == SSA_NAME);
+		      DR_OFFSET (newdr) = ssize_int (0);
-		      arg = SSA_NAME_VAR (arg);
+		      DR_STEP (newdr) = step;
-		      if (arg == loop->simduid
+		      DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
-			  /* For now.  */
+		      DR_STEP_ALIGNMENT (newdr) = highest_pow2_factor (step);
-			  && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
+		      /* Mark as simd-lane access.  */
-			{
+		      tree arg2 = gimple_call_arg (def, 1);
-			  DR_OFFSET (newdr) = ssize_int (0);
+		      newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
-			  DR_STEP (newdr) = step;
+		      free_data_ref (dr);
-			  DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
+		      datarefs->safe_push (newdr);
-			  DR_STEP_ALIGNMENT (newdr)
+		      return opt_result::success ();
-			    = highest_pow2_factor (step);
-			  /* Mark as simd-lane access.  */
-			  newdr->aux = (void *)-1;
-			  free_data_ref (dr);
-			  datarefs->safe_push (newdr);
-			  return opt_result::success ();
-			}
 		    }
 		}
 	    }
 	}
 free_data_ref (newdr);
 4- vect_analyze_drs_access(): check that ref_stmt.step is ok.
 */
 opt_result
-vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf)
+vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
 {
-struct loop *loop = NULL;
+class loop *loop = NULL;
 unsigned int i;
 struct data_reference *dr;
 tree scalar_type;
 DUMP_VECT_SCOPE ("vect_analyze_data_refs");
 					     stmt_info->stmt);
 	    }
 }
 /* See if this was detected as SIMD lane access.  */
-if (dr->aux == (void *)-1)
+if (dr->aux == (void *)-1
+	  || dr->aux == (void *)-2
+	  || dr->aux == (void *)-3
+	  || dr->aux == (void *)-4)
 	{
 	  if (nested_in_vect_loop_p (loop, stmt_info))
 	    return opt_result::failure_at (stmt_info->stmt,
 					   "not vectorized:"
 					   " data ref analysis failed: %G",
 					   stmt_info->stmt);
-	  STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) = true;
+	  STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
+	    = -(uintptr_t) dr->aux;
 	}
 tree base = get_base_address (DR_REF (dr));
 if (base && VAR_P (base) && DECL_NONALIASED (base))
 	{
 	  && DR_STEP (dr)
 	  && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
 	{
 	  if (nested_in_vect_loop_p (loop, stmt_info))
 	    return opt_result::failure_at (stmt_info->stmt,
-					   "not vectorized:"
+					   "not vectorized: "
 					   "not suitable for strided load %G",
 					   stmt_info->stmt);
 	  STMT_VINFO_STRIDED_P (stmt_info) = true;
 	}
 			     STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
 	}
 /* Set vectype for STMT.  */
 scalar_type = TREE_TYPE (DR_REF (dr));
-STMT_VINFO_VECTYPE (stmt_info)
+tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
-	= get_vectype_for_scalar_type (scalar_type);
+if (!vectype)
-if (!STMT_VINFO_VECTYPE (stmt_info))
 {
 if (dump_enabled_p ())
 {
 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 "not vectorized: no vectype for stmt: %G",
 	      /* No vector type is fine, the ref can still participate
 	         in dependence analysis, we just can't vectorize it.  */
 	      STMT_VINFO_VECTORIZABLE (stmt_info) = false;
 	      continue;
 	    }
+	  if (fatal)
+	    *fatal = false;
 	  return opt_result::failure_at (stmt_info->stmt,
 					 "not vectorized:"
 					 " no vectype for stmt: %G"
 					 " scalar_type: %T\n",
 					 stmt_info->stmt, scalar_type);
 else
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "got vectype for stmt: %G%T\n",
-			     stmt_info->stmt, STMT_VINFO_VECTYPE (stmt_info));
+			     stmt_info->stmt, vectype);
 	}
 /* Adjust the minimal vectorization factor according to the
 	 vector type.  */
-vf = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
+vf = TYPE_VECTOR_SUBPARTS (vectype);
 *min_vf = upper_bound (*min_vf, vf);
+/* Leave the BB vectorizer to pick the vector type later, based on
+	 the final dataref group size and SLP node size.  */
+if (is_a <loop_vec_info> (vinfo))
+	STMT_VINFO_VECTYPE (stmt_info) = vectype;
 if (gatherscatter != SG_NONE)
 	{
 	  gather_scatter_info gs_info;
 	  if (!vect_check_gather_scatter (stmt_info,
 					  as_a <loop_vec_info> (vinfo),
 					  &gs_info)
-	      || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset)))
+	      || !get_vectype_for_scalar_type (vinfo,
-	    return opt_result::failure_at
+					       TREE_TYPE (gs_info.offset)))
-	      (stmt_info->stmt,
+	    {
-	       (gatherscatter == GATHER) ?
+	      if (fatal)
-	       "not vectorized: not suitable for gather load %G" :
+		*fatal = false;
-	       "not vectorized: not suitable for scatter store %G",
+	      return opt_result::failure_at
-	       stmt_info->stmt);
+			(stmt_info->stmt,
+			 (gatherscatter == GATHER)
+			 ? "not vectorized: not suitable for gather load %G"
+			 : "not vectorized: not suitable for scatter store %G",
+			 stmt_info->stmt);
+	    }
 	  STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
 	}
 }
 /* We used to stop processing and prune the list here.  Verify we no
 int misalign = DR_MISALIGNMENT (dr_info);
 if (misalign == DR_MISALIGNMENT_UNKNOWN)
 mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (name));
 else
 set_ptr_info_alignment (SSA_NAME_PTR_INFO (name),
-			    DR_TARGET_ALIGNMENT (dr_info), misalign);
+			    known_alignment (DR_TARGET_ALIGNMENT (dr_info)),
+			    misalign);
 }
 /* Function vect_create_addr_base_for_vector_ref.
 Create an expression that computes the address of the first memory location
 tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
 innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
 tree data_ref_base = unshare_expr (drb->base_address);
-tree base_offset = unshare_expr (drb->offset);
+tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true));
 tree init = unshare_expr (drb->init);
 if (loop_vinfo)
 base_name = get_name (data_ref_base);
 else
 3. Return the pointer.  */
 tree
 vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
-			  struct loop *at_loop, tree offset,
+			  class loop *at_loop, tree offset,
 			  tree *initial_address, gimple_stmt_iterator *gsi,
 			  gimple **ptr_incr, bool only_init,
 			  tree byte_offset, tree iv_step)
 {
 const char *base_name;
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
-struct loop *loop = NULL;
+class loop *loop = NULL;
 bool nested_in_vect_loop = false;
-struct loop *containing_loop = NULL;
+class loop *containing_loop = NULL;
 tree aggr_ptr_type;
 tree aggr_ptr;
 tree new_temp;
 gimple_seq new_stmt_list = NULL;
 edge pe = NULL;
 tree
 vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 tree *realignment_token,
 			enum dr_alignment_support alignment_support_scheme,
 			tree init_addr,
-			struct loop **at_loop)
+			class loop **at_loop)
 {
 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
 dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
 struct data_reference *dr = dr_info->dr;
-struct loop *loop = NULL;
+class loop *loop = NULL;
 edge pe = NULL;
 tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
 tree vec_dest;
 gimple *inc;
 tree ptr;
 gphi *phi_stmt;
 tree msq = NULL_TREE;
 gimple_seq stmts = NULL;
 bool compute_in_loop = false;
 bool nested_in_vect_loop = false;
-struct loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
+class loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
-struct loop *loop_for_initial_load = NULL;
+class loop *loop_for_initial_load = NULL;
 if (loop_vinfo)
 {
 loop = LOOP_VINFO_LOOP (loop_vinfo);
 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
 				      &init_addr, NULL, &inc, true);
 if (TREE_CODE (ptr) == SSA_NAME)
 	new_temp = copy_ssa_name (ptr);
 else
 	new_temp = make_ssa_name (TREE_TYPE (ptr));
-unsigned int align = DR_TARGET_ALIGNMENT (dr_info);
+poly_uint64 align = DR_TARGET_ALIGNMENT (dr_info);
+tree type = TREE_TYPE (ptr);
 new_stmt = gimple_build_assign
 		   (new_temp, BIT_AND_EXPR, ptr,
-		    build_int_cst (TREE_TYPE (ptr), -(HOST_WIDE_INT) align));
+		    fold_build2 (MINUS_EXPR, type,
+				 build_int_cst (type, 0),
+				 build_int_cst (type, align)));
 new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
 gcc_assert (!new_bb);
 data_ref
 	= build2 (MEM_REF, TREE_TYPE (vec_dest), new_temp,
 		  build_int_cst (reference_alias_ptr_type (DR_REF (dr)), 0));
 DR_GROUP_GAP is the number of steps in elements from the previous
 access (if there is no gap DR_GROUP_GAP is 1).  We skip loads that
 correspond to the gaps.  */
 if (next_stmt_info != first_stmt_info
 	  && gap_count < DR_GROUP_GAP (next_stmt_info))
-{
+	{
-gap_count++;
+	  gap_count++;
-continue;
+	  continue;
-}
+	}
-while (next_stmt_info)
+/* ???  The following needs cleanup after the removal of
+DR_GROUP_SAME_DR_STMT.  */
+if (next_stmt_info)
 {
 	  stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref);
 	  /* We assume that if VEC_STMT is not NULL, this is a case of multiple
 	     copies, and we put the new vector statement in the first available
 	     RELATED_STMT.  */
 	  if (!STMT_VINFO_VEC_STMT (next_stmt_info))
 	    STMT_VINFO_VEC_STMT (next_stmt_info) = new_stmt_info;
 	  else
 {
-	      if (!DR_GROUP_SAME_DR_STMT (next_stmt_info))
+	      stmt_vec_info prev_stmt_info
-{
+		= STMT_VINFO_VEC_STMT (next_stmt_info);
-		  stmt_vec_info prev_stmt_info
+	      stmt_vec_info rel_stmt_info
-		    = STMT_VINFO_VEC_STMT (next_stmt_info);
+		= STMT_VINFO_RELATED_STMT (prev_stmt_info);
-		  stmt_vec_info rel_stmt_info
+	      while (rel_stmt_info)
-		    = STMT_VINFO_RELATED_STMT (prev_stmt_info);
+		{
-		  while (rel_stmt_info)
+		  prev_stmt_info = rel_stmt_info;
-		    {
+		  rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info);
-		      prev_stmt_info = rel_stmt_info;
+		}
-		      rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info);
-		    }
+	      STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-		  STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
-}
 }
 	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
 	  gap_count = 1;
-	  /* If NEXT_STMT_INFO accesses the same DR as the previous statement,
-	     put the same TMP_DATA_REF as its vectorized statement; otherwise
-	     get the next data-ref from RESULT_CHAIN.  */
-	  if (!next_stmt_info || !DR_GROUP_SAME_DR_STMT (next_stmt_info))
-	    break;
 }
 }
 }
 /* Function vect_force_dr_alignment_p.
 Returns whether the alignment of a DECL can be forced to be aligned
 on ALIGNMENT bit boundary.  */
 bool
-vect_can_force_dr_alignment_p (const_tree decl, unsigned int alignment)
+vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
 {
 if (!VAR_P (decl))
 return false;
 if (decl_in_symtab_p (decl)
 && !symtab_node::get (decl)->can_increase_alignment_p ())
 return false;
 if (TREE_STATIC (decl))
-return (alignment <= MAX_OFILE_ALIGNMENT);
+return (known_le (alignment,
+		      (unsigned HOST_WIDE_INT) MAX_OFILE_ALIGNMENT));
 else
-return (alignment <= MAX_STACK_ALIGNMENT);
+return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
 }
 /* Return whether the data reference DR_INFO is supported with respect to its
 alignment.
 data_reference *dr = dr_info->dr;
 stmt_vec_info stmt_info = dr_info->stmt;
 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 machine_mode mode = TYPE_MODE (vectype);
 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
-struct loop *vect_loop = NULL;
+class loop *vect_loop = NULL;
 bool nested_in_vect_loop = false;
 if (aligned_access_p (dr_info) && !check_aligned_accesses)
 return dr_aligned;

Mercurial > hg > CbC > CbC_gcc

comparison gcc/tree-vect-data-refs.c @ 145:1830386684a0