CbC/CbC_gcc: gcc/config/i386/sse.md comparison

comparison gcc/config/i386/sse.md @ 55:77e2b8dfacca gcc-4.4.5

update it from 4.4.3 to 4.5.0

author	ryoma <e075725@ie.u-ryukyu.ac.jp>
date	Fri, 12 Feb 2010 23:39:51 +0900
parents	3bfb6c00c1e0
children	b7f97abdc517

comparison

equal deleted inserted replaced

-:c156f1bd5cd9
+:77e2b8dfacca
 ;; You should have received a copy of the GNU General Public License
 ;; along with GCC; see the file COPYING3.  If not see
 ;; <http://www.gnu.org/licenses/>.
-;; 16 byte integral modes handled by SSE, minus TImode, which gets
+;; 16 byte integral modes handled by SSE
-;; special-cased for TARGET_64BIT.
 (define_mode_iterator SSEMODEI [V16QI V8HI V4SI V2DI])
 ;; All 16-byte vector modes handled by SSE
 (define_mode_iterator SSEMODE [V16QI V8HI V4SI V2DI V4SF V2DF])
+(define_mode_iterator SSEMODE16 [V16QI V8HI V4SI V2DI V1TI V4SF V2DF])
 ;; 32 byte integral vector modes handled by AVX
 (define_mode_iterator AVX256MODEI [V32QI V16HI V8SI V4DI])
 ;; All 32-byte vector modes handled by AVX
 ;; All DI vector modes handled by AVX
 (define_mode_iterator AVXMODEDI [V4DI V2DI])
 ;; All vector modes handled by AVX
-(define_mode_iterator AVXMODE [V16QI V8HI V4SI V2DI V4SF V2DF V32QI V16HI V8SI V4DI V8SF V4DF])
+(define_mode_iterator AVXMODE
+[V16QI V8HI V4SI V2DI V4SF V2DF V32QI V16HI V8SI V4DI V8SF V4DF])
+(define_mode_iterator AVXMODE16
+[V16QI V8HI V4SI V2DI V1TI V4SF V2DF V32QI V16HI V8SI V4DI V8SF V4DF])
 ;; Mix-n-match
 (define_mode_iterator SSEMODE12 [V16QI V8HI])
 (define_mode_iterator SSEMODE24 [V8HI V4SI])
 (define_mode_iterator SSEMODE14 [V16QI V4SI])
 (define_mode_iterator SSEMODE124 [V16QI V8HI V4SI])
 (define_mode_iterator SSEMODE248 [V8HI V4SI V2DI])
 (define_mode_iterator SSEMODE1248 [V16QI V8HI V4SI V2DI])
 (define_mode_iterator SSEMODEF4 [SF DF V4SF V2DF])
+(define_mode_iterator FMA4MODEF4 [V8SF V4DF])
 (define_mode_iterator SSEMODEF2P [V4SF V2DF])
 (define_mode_iterator AVX256MODEF2P [V8SF V4DF])
 (define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
+(define_mode_iterator AVX256MODE24P [V8SI V8SF V4DI V4DF])
 (define_mode_iterator AVX256MODE4P [V4DI V4DF])
 (define_mode_iterator AVX256MODE8P [V8SI V8SF])
 (define_mode_iterator AVXMODEF2P [V4SF V2DF V8SF V4DF])
 (define_mode_iterator AVXMODEF4P [V4SF V4DF])
+(define_mode_iterator AVXMODEFDP [V2DF V4DF])
+(define_mode_iterator AVXMODEFSP [V4SF V8SF])
 (define_mode_iterator AVXMODEDCVTDQ2PS [V4SF V8SF])
 (define_mode_iterator AVXMODEDCVTPS2DQ [V4SI V8SI])
 ;; Int-float size matches
 (define_mode_iterator SSEMODE4S [V4SF V4SI])
 (define_mode_iterator SSEMODE2D [V2DF V2DI])
 ;; Modes handled by integer vcond pattern
 (define_mode_iterator SSEMODE124C8 [V16QI V8HI V4SI
-				    (V2DI "TARGET_SSE4_2 || TARGET_SSE5")])
+				    (V2DI "TARGET_SSE4_2")])
+;; Modes handled by vec_extract_even/odd pattern.
+(define_mode_iterator SSEMODE_EO
+[(V4SF "TARGET_SSE")
+(V2DF "TARGET_SSE2")
+(V2DI "TARGET_SSE2") (V4SI "TARGET_SSE2")
+(V8HI "TARGET_SSE2") (V16QI "TARGET_SSE2")
+(V4DF "TARGET_AVX") (V8SF "TARGET_AVX")])
 ;; Mapping from float mode to required SSE level
 (define_mode_attr sse [(SF "sse") (DF "sse2") (V4SF "sse") (V2DF "sse2")])
 ;; Mapping from integer vector mode to mnemonic suffix
 (define_mode_attr ssevecsize [(V16QI "b") (V8HI "w") (V4SI "d") (V2DI "q")])
-;; Mapping of the sse5 suffix
+;; Mapping of the fma4 suffix
+(define_mode_attr fma4modesuffixf4 [(V8SF "ps") (V4DF "pd")])
+(define_mode_attr ssemodesuffixf2s [(SF "ss") (DF "sd")
+				    (V4SF "ss") (V2DF "sd")])
+;; Mapping of the avx suffix
 (define_mode_attr ssemodesuffixf4 [(SF "ss") (DF "sd")
 				   (V4SF "ps") (V2DF "pd")])
-(define_mode_attr ssemodesuffixf2s [(SF "ss") (DF "sd")
-				    (V4SF "ss") (V2DF "sd")])
 (define_mode_attr ssemodesuffixf2c [(V4SF "s") (V2DF "d")])
-;; Mapping of the max integer size for sse5 rotate immediate constraint
+(define_mode_attr ssescalarmodesuffix2s [(V4SF "ss") (V4SI "d")])
+;; Mapping of the max integer size for xop rotate immediate constraint
 (define_mode_attr sserotatemax [(V16QI "7") (V8HI "15") (V4SI "31") (V2DI "63")])
 ;; Mapping of vector modes back to the scalar modes
 (define_mode_attr ssescalarmode [(V4SF "SF") (V2DF "DF")
 				 (V16QI "QI") (V8HI "HI")
 				 (V4SI "SI") (V2DI "DI")])
 ;; Mapping of vector modes to a vector mode of double size
-(define_mode_attr ssedoublesizemode [(V2DF "V4DF") (V2DI "V4DI")
+(define_mode_attr ssedoublesizemode
-				     (V4SF "V8SF") (V4SI "V8SI")])
+[(V2DF "V4DF") (V2DI "V4DI") (V4SF "V8SF") (V4SI "V8SI")
+(V8HI "V16HI") (V16QI "V32QI")
+(V4DF "V8DF") (V8SF "V16SF")
+(V4DI "V8DI") (V8SI "V16SI") (V16HI "V32HI") (V32QI "V64QI")])
 ;; Number of scalar elements in each vector type
-(define_mode_attr ssescalarnum [(V4SF "4") (V2DF "2")
+(define_mode_attr ssescalarnum
-				(V16QI "16") (V8HI "8")
+[(V4SF "4") (V2DF "2") (V16QI "16") (V8HI "8") (V4SI "4") (V2DI "2")
-				(V4SI "4") (V2DI "2")])
+(V8SF "8") (V4DF "4") (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")])
 ;; Mapping for AVX
 (define_mode_attr avxvecmode
-[(V16QI "TI") (V8HI "TI") (V4SI "TI") (V2DI "TI") (V4SF "V4SF")
+[(V16QI "TI") (V8HI "TI") (V4SI "TI") (V2DI "TI") (V1TI "TI")
-(V2DF "V2DF") (V32QI "OI") (V16HI "OI") (V8SI "OI") (V4DI "OI")
+(V4SF "V4SF") (V8SF "V8SF") (V2DF "V2DF") (V4DF "V4DF")
-(V8SF "V8SF") (V4DF "V4DF")])
+(V32QI "OI") (V16HI "OI") (V8SI "OI") (V4DI "OI")])
 (define_mode_attr avxvecpsmode
 [(V16QI "V4SF") (V8HI "V4SF") (V4SI "V4SF") (V2DI "V4SF")
 (V32QI "V8SF") (V16HI "V8SF") (V8SI "V8SF") (V4DI "V8SF")])
 (define_mode_attr avxhalfvecmode
-[(V4SF "V2SF") (V32QI "V16QI")  (V16HI "V8HI") (V8SI "V4SI")
+[(V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
-(V4DI "V2DI") (V8SF "V4SF") (V4DF "V2DF")])
+(V8SF "V4SF") (V4DF "V2DF")
+(V16QI  "V8QI") (V8HI  "V4HI") (V4SI "V2SI") (V4SF "V2SF")])
 (define_mode_attr avxscalarmode
-[(V16QI "QI") (V8HI "HI") (V4SI "SI") (V4SF "SF") (V2DF "DF")
+[(V16QI "QI") (V8HI  "HI") (V4SI "SI") (V2DI "DI") (V4SF "SF") (V2DF "DF")
-(V8SF "SF") (V4DF "DF")])
+(V32QI "QI") (V16HI "HI") (V8SI "SI") (V4DI "DI") (V8SF "SF") (V4DF "DF")])
 (define_mode_attr avxcvtvecmode
 [(V4SF "V4SI") (V8SF "V8SI") (V4SI "V4SF") (V8SI "V8SF")])
 (define_mode_attr avxpermvecmode
 [(V2DF "V2DI") (V4SF "V4SI") (V4DF "V4DI") (V8SF "V8SI")])
 (define_mode_attr avxmodesuffixf2c
-[(V4SF "s") (V2DF "d") (V8SF "s") (V4DF "d")])
+[(V4SF "s") (V2DF "d") (V8SI "s") (V8SF "s") (V4DI "d") (V4DF "d")])
 (define_mode_attr avxmodesuffixp
 [(V2DF "pd") (V4SI "si") (V4SF "ps") (V8SF "ps") (V8SI "si")
 (V4DF "pd")])
-(define_mode_attr avxmodesuffixs
-[(V16QI "b") (V8HI "w") (V4SI "d")])
 (define_mode_attr avxmodesuffix
 [(V16QI "") (V32QI "256") (V4SI "") (V4SF "") (V2DF "")
 (V8SI "256") (V8SF "256") (V4DF "256")])
 ;; Mapping of immediate bits for blend instructions
 (define_mode_attr blendbits
 [(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
-;; Mapping of immediate bits for vpermil instructions
-(define_mode_attr vpermilbits
-[(V8SF "255") (V4SF "255") (V4DF "15") (V2DF "3")])
 ;; Mapping of immediate bits for pinsr instructions
 (define_mode_attr pinsrbits [(V16QI "32768") (V8HI "128") (V4SI "8")])
 ;; Patterns whose name begins with "sse{,2,3}_" are invoked by intrinsics.
 ix86_expand_vector_move (<MODE>mode, operands);
 DONE;
 })
 (define_insn "*avx_mov<mode>_internal"
-[(set (match_operand:AVXMODE 0 "nonimmediate_operand" "=x,x ,m")
+[(set (match_operand:AVXMODE16 0 "nonimmediate_operand" "=x,x ,m")
-	(match_operand:AVXMODE 1 "nonimmediate_or_sse_const_operand"  "C ,xm,x"))]
+	(match_operand:AVXMODE16 1 "nonimmediate_or_sse_const_operand"  "C ,xm,x"))]
 "TARGET_AVX
 && (register_operand (operands[0], <MODE>mode)
 || register_operand (operands[1], <MODE>mode))"
 {
 switch (which_alternative)
 ;; All of these patterns are enabled for SSE1 as well as SSE2.
 ;; This is essential for maintaining stable calling conventions.
 (define_expand "mov<mode>"
-[(set (match_operand:SSEMODE 0 "nonimmediate_operand" "")
+[(set (match_operand:SSEMODE16 0 "nonimmediate_operand" "")
-	(match_operand:SSEMODE 1 "nonimmediate_operand" ""))]
+	(match_operand:SSEMODE16 1 "nonimmediate_operand" ""))]
 "TARGET_SSE"
 {
 ix86_expand_vector_move (<MODE>mode, operands);
 DONE;
 })
 (define_insn "*mov<mode>_internal"
-[(set (match_operand:SSEMODE 0 "nonimmediate_operand" "=x,x ,m")
+[(set (match_operand:SSEMODE16 0 "nonimmediate_operand" "=x,x ,m")
-	(match_operand:SSEMODE 1 "nonimmediate_or_sse_const_operand"  "C ,xm,x"))]
+	(match_operand:SSEMODE16 1 "nonimmediate_or_sse_const_operand"  "C ,xm,x"))]
 "TARGET_SSE
 && (register_operand (operands[0], <MODE>mode)
 || register_operand (operands[1], <MODE>mode))"
 {
 switch (which_alternative)
 	 Assemble the 64-bit DImode value in an xmm register.  */
 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode),
 				  gen_rtx_SUBREG (SImode, operands[1], 0)));
 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode),
 				  gen_rtx_SUBREG (SImode, operands[1], 4)));
-emit_insn (gen_sse2_punpckldq (operands[0], operands[0], operands[2]));
+emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
+					     operands[2]));
 }
 else if (memory_operand (operands[1], DImode))
-emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]), operands[1], const0_rtx));
+emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]),
+	     			  operands[1], const0_rtx));
 else
 gcc_unreachable ();
 })
 (define_split
 [(set (match_operand:V4SF 0 "register_operand" "")
 	(match_operand:V4SF 1 "zero_extended_scalar_load_operand" ""))]
 ix86_expand_push (<MODE>mode, operands[0]);
 DONE;
 })
 (define_expand "push<mode>1"
-[(match_operand:SSEMODE 0 "register_operand" "")]
+[(match_operand:SSEMODE16 0 "register_operand" "")]
 "TARGET_SSE"
 {
 ix86_expand_push (<MODE>mode, operands[0]);
 DONE;
 })
 ix86_expand_vector_move_misalign (<MODE>mode, operands);
 DONE;
 })
 (define_expand "movmisalign<mode>"
-[(set (match_operand:SSEMODE 0 "nonimmediate_operand" "")
+[(set (match_operand:SSEMODE16 0 "nonimmediate_operand" "")
-	(match_operand:SSEMODE 1 "nonimmediate_operand" ""))]
+	(match_operand:SSEMODE16 1 "nonimmediate_operand" ""))]
 "TARGET_SSE"
 {
 ix86_expand_vector_move_misalign (<MODE>mode, operands);
 DONE;
 })
 	  UNSPEC_MOVU))]
 "AVX_VEC_FLOAT_MODE_P (<MODE>mode)
 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "vmovup<avxmodesuffixf2c>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssemov")
+(set_attr "movu" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse2_movq128"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	  UNSPEC_MOVU))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)
 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "movup<ssemodesuffixf2c>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssemov")
+(set_attr "movu" "1")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_movdqu<avxmodesuffix>"
 [(set (match_operand:AVXMODEQI 0 "nonimmediate_operand" "=x,m")
 	(unspec:AVXMODEQI
 	  [(match_operand:AVXMODEQI 1 "nonimmediate_operand" "xm,x")]
 	  UNSPEC_MOVU))]
 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "vmovdqu\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssemov")
+(set_attr "movu" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "sse2_movdqu"
 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
 	(unspec:V16QI [(match_operand:V16QI 1 "nonimmediate_operand" "xm,x")]
 		      UNSPEC_MOVU))]
 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "movdqu\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssemov")
+(set_attr "movu" "1")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_insn "avx_movnt<mode>"
 [(set (match_operand:AVXMODEF2P 0 "memory_operand" "=m")
 [(set (match_operand:V2DI 0 "memory_operand" "=m")
 	(unspec:V2DI [(match_operand:V2DI 1 "register_operand" "x")]
 		     UNSPEC_MOVNT))]
 "TARGET_SSE2"
 "movntdq\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_insn "sse2_movntsi"
 [(set (match_operand:SI 0 "memory_operand" "=m")
 	(unspec:SI [(match_operand:SI 1 "register_operand" "r")]
 		   UNSPEC_MOVNT))]
 "TARGET_SSE2"
 "movnti\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
+(set_attr "prefix_data16" "0")
 (set_attr "mode" "V2DF")])
 (define_insn "avx_lddqu<avxmodesuffix>"
 [(set (match_operand:AVXMODEQI 0 "register_operand" "=x")
 	(unspec:AVXMODEQI
 	  [(match_operand:AVXMODEQI 1 "memory_operand" "m")]
 	  UNSPEC_LDDQU))]
 "TARGET_AVX"
 "vlddqu\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
+(set_attr "movu" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "sse3_lddqu"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(unspec:V16QI [(match_operand:V16QI 1 "memory_operand" "m")]
 		      UNSPEC_LDDQU))]
 "TARGET_SSE3"
 "lddqu\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
+(set_attr "movu" "1")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix_rep" "1")
 (set_attr "mode" "TI")])
 ; Expand patterns for non-temporal stores.  At the moment, only those
 ; that directly map to insns are defined; it would be possible to
 		  (match_operand:V8SF 2 "nonimmediate_operand" "")))]
 "TARGET_AVX"
 {
 ix86_fixup_binary_operands_no_copy (DIV, V8SFmode, operands);
-if (TARGET_SSE_MATH && TARGET_RECIP && !optimize_size
+if (TARGET_SSE_MATH && TARGET_RECIP && !optimize_insn_for_size_p ()
 && flag_finite_math_only && !flag_trapping_math
 && flag_unsafe_math_optimizations)
 {
 ix86_emit_swdivsf (operands[0], operands[1],
 			 operands[2], V8SFmode);
 	(unspec:V4SF
 	  [(match_operand:V4SF 1 "nonimmediate_operand" "xm")] UNSPEC_RCP))]
 "TARGET_SSE"
 "%vrcpps\t{%1, %0|%0, %1}"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "rcp")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "V4SF")])
 (define_insn "*avx_vmrcpv4sf2"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	  (match_operand:V4SF 2 "register_operand" "0")
 	  (const_int 1)))]
 "TARGET_SSE"
 "rcpss\t{%1, %0|%0, %1}"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "rcp")
 (set_attr "mode" "SF")])
 (define_expand "sqrtv8sf2"
 [(set (match_operand:V8SF 0 "register_operand" "")
 	(sqrt:V8SF (match_operand:V8SF 1 "nonimmediate_operand" "")))]
 "TARGET_AVX"
 {
-if (TARGET_SSE_MATH && TARGET_RECIP && !optimize_size
+if (TARGET_SSE_MATH && TARGET_RECIP && !optimize_insn_for_size_p ()
 && flag_finite_math_only && !flag_trapping_math
 && flag_unsafe_math_optimizations)
 {
 ix86_emit_swsqrtsf (operands[0], operands[1], V8SFmode, 0);
 DONE;
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(sqrt:V4SF (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
 "TARGET_SSE"
 "%vsqrtps\t{%1, %0|%0, %1}"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "sqrt")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "V4SF")])
 (define_insn "sqrtv4df2"
 [(set (match_operand:V4DF 0 "register_operand" "=x")
 	  (match_operand:SSEMODEF2P 2 "register_operand" "0")
 	  (const_int 1)))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
 "sqrts<ssemodesuffixf2c>\t{%1, %0|%0, %1}"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "sqrt")
 (set_attr "mode" "<ssescalarmode>")])
 (define_expand "rsqrtv8sf2"
 [(set (match_operand:V8SF 0 "register_operand" "")
 	(unspec:V8SF
 	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "xm"))
 	 (match_dup 1)
 	 (const_int 1)))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
 "<maxminfprefix>s<ssemodesuffixf2c>\t{%2, %0|%0, %2}"
-[(set_attr "type" "sse")
+[(set_attr "type" "sseadd")
 (set_attr "mode" "<ssescalarmode>")])
 ;; These versions of the min/max patterns implement exactly the operations
 ;;   min = (op1 < op2 ? op1 : op2)
 ;;   max = (!(op1 < op2) ? op1 : op2)
 	  (minus:V2DF (match_dup 1) (match_dup 2))
 	  (const_int 2)))]
 "TARGET_SSE3"
 "addsubpd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseadd")
+(set_attr "atom_unit" "complex")
 (set_attr "mode" "V2DF")])
 (define_insn "avx_h<plusminus_insn>v4df3"
 [(set (match_operand:V4DF 0 "register_operand" "=x")
 	(vec_concat:V4DF
 	      (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:SF (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSE3"
 "h<plusminus_mnemonic>ps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_rep" "1")
 (set_attr "mode" "V4SF")])
 (define_insn "*avx_h<plusminus_insn>v2df3"
 [(set (match_operand:V2DF 0 "register_operand" "=x")
 	   (match_operand:SI 3 "const_0_to_31_operand" "n")]
 	  UNSPEC_PCMP))]
 "TARGET_AVX"
 "vcmpp<avxmodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssecmp")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_cmps<ssemodesuffixf2c><mode>3"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "")
 	 (match_dup 1)
 	 (const_int 1)))]
 "TARGET_AVX"
 "vcmps<ssemodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssecmp")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<ssescalarmode>")])
 ;; We don't promote 128bit vector compare intrinsics. But vectorizer
 ;; may generate 256bit vector compare instructions.
 		 (match_operand:AVXMODEF2P 2 "nonimmediate_operand" "xm")]))]
 "AVX_VEC_FLOAT_MODE_P (<MODE>mode)"
 "vcmp%D3p<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "ssecmp")
 (set_attr "prefix" "vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "<sse>_maskcmp<mode>3"
 [(set (match_operand:SSEMODEF4 0 "register_operand" "=x")
 	(match_operator:SSEMODEF4 3 "sse_comparison_operator"
 		[(match_operand:SSEMODEF4 1 "register_operand" "0")
 		 (match_operand:SSEMODEF4 2 "nonimmediate_operand" "xm")]))]
-"(SSE_FLOAT_MODE_P (<MODE>mode) || SSE_VEC_FLOAT_MODE_P (<MODE>mode))
+"!TARGET_XOP
-&& !TARGET_SSE5"
+&& (SSE_FLOAT_MODE_P (<MODE>mode) || SSE_VEC_FLOAT_MODE_P (<MODE>mode))"
 "cmp%D3<ssemodesuffixf4>\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecmp")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<MODE>")])
 (define_insn "<sse>_vmmaskcmp<mode>3"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	(vec_merge:SSEMODEF2P
 	 (match_operator:SSEMODEF2P 3 "sse_comparison_operator"
 		[(match_operand:SSEMODEF2P 1 "register_operand" "0")
 		 (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "xm")])
 	 (match_dup 1)
 	 (const_int 1)))]
-"SSE_VEC_FLOAT_MODE_P (<MODE>mode) && !TARGET_SSE5"
+"SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
 "cmp%D3s<ssemodesuffixf2c>\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecmp")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<ssescalarmode>")])
 (define_insn "<sse>_comi"
 [(set (reg:CCFP FLAGS_REG)
 	(compare:CCFP
 	    (parallel [(const_int 0)]))))]
 "SSE_FLOAT_MODE_P (<MODE>mode)"
 "%vcomis<ssemodefsuffix>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecomi")
 (set_attr "prefix" "maybe_vex")
+(set_attr "prefix_rep" "0")
+(set (attr "prefix_data16")
+	(if_then_else (eq_attr "mode" "DF")
+		      (const_string "1")
+		      (const_string "0")))
 (set_attr "mode" "<MODE>")])
 (define_insn "<sse>_ucomi"
 [(set (reg:CCFPU FLAGS_REG)
 	(compare:CCFPU
 	    (parallel [(const_int 0)]))))]
 "SSE_FLOAT_MODE_P (<MODE>mode)"
 "%vucomis<ssemodefsuffix>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecomi")
 (set_attr "prefix" "maybe_vex")
+(set_attr "prefix_rep" "0")
+(set (attr "prefix_data16")
+	(if_then_else (eq_attr "mode" "DF")
+		      (const_string "1")
+		      (const_string "0")))
 (set_attr "mode" "<MODE>")])
 (define_expand "vcond<mode>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "")
 (if_then_else:SSEMODEF2P
 [(set_attr "type" "sselog")
 (set_attr "mode" "<MODE>")])
 (define_expand "<code><mode>3"
 [(set (match_operand:AVX256MODEF2P 0 "register_operand" "")
-	(plogic:AVX256MODEF2P
+	(any_logic:AVX256MODEF2P
 	  (match_operand:AVX256MODEF2P 1 "nonimmediate_operand" "")
 	  (match_operand:AVX256MODEF2P 2 "nonimmediate_operand" "")))]
 "AVX256_VEC_FLOAT_MODE_P (<MODE>mode)"
 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
 (define_insn "*avx_<code><mode>3"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
-	(plogic:AVXMODEF2P
+	(any_logic:AVXMODEF2P
 	  (match_operand:AVXMODEF2P 1 "nonimmediate_operand" "%x")
 	  (match_operand:AVXMODEF2P 2 "nonimmediate_operand" "xm")))]
 "AVX_VEC_FLOAT_MODE_P (<MODE>mode)
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"v<plogicprefix>p<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
+"v<logicprefix>p<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])
 (define_expand "<code><mode>3"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(plogic:SSEMODEF2P
+	(any_logic:SSEMODEF2P
 	  (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "")
 	  (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "")))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
 (define_insn "*<code><mode>3"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
-	(plogic:SSEMODEF2P
+	(any_logic:SSEMODEF2P
 	  (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%0")
 	  (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "xm")))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"<plogicprefix>p<ssemodesuffixf2c>\t{%2, %0|%0, %2}"
+"<logicprefix>p<ssemodesuffixf2c>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "mode" "<MODE>")])
+(define_expand "copysign<mode>3"
+[(set (match_dup 4)
+	(and:SSEMODEF2P
+	  (not:SSEMODEF2P (match_dup 3))
+	  (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "")))
+(set (match_dup 5)
+	(and:SSEMODEF2P (match_dup 3)
+			(match_operand:SSEMODEF2P 2 "nonimmediate_operand" "")))
+(set (match_operand:SSEMODEF2P 0 "register_operand" "")
+	(ior:SSEMODEF2P (match_dup 4) (match_dup 5)))]
+"SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
+{
+operands[3] = ix86_build_signbit_mask (<ssescalarmode>mode, 1, 0);
+operands[4] = gen_reg_rtx (<MODE>mode);
+operands[5] = gen_reg_rtx (<MODE>mode);
+})
 ;; Also define scalar versions.  These are used for abs, neg, and
 ;; conditional move.  Using subregs into vector modes causes register
 ;; allocation lossage.  These patterns do not allow memory operands
 ;; because the native instructions read the full 128-bits.
 [(set_attr "type" "sselog")
 (set_attr "mode" "<ssevecmode>")])
 (define_insn "*avx_<code><mode>3"
 [(set (match_operand:MODEF 0 "register_operand" "=x")
-	(plogic:MODEF
+	(any_logic:MODEF
 	  (match_operand:MODEF 1 "register_operand" "x")
 	  (match_operand:MODEF 2 "register_operand" "x")))]
 "AVX_FLOAT_MODE_P (<MODE>mode)"
-"v<plogicprefix>p<ssemodefsuffix>\t{%2, %1, %0|%0, %1, %2}"
+"v<logicprefix>p<ssemodefsuffix>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<ssevecmode>")])
 (define_insn "*<code><mode>3"
 [(set (match_operand:MODEF 0 "register_operand" "=x")
-	(plogic:MODEF
+	(any_logic:MODEF
 	  (match_operand:MODEF 1 "register_operand" "0")
 	  (match_operand:MODEF 2 "register_operand" "x")))]
 "SSE_FLOAT_MODE_P (<MODE>mode)"
-"<plogicprefix>p<ssemodefsuffix>\t{%2, %0|%0, %2}"
+"<logicprefix>p<ssemodefsuffix>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "mode" "<ssevecmode>")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
-;; SSE5 floating point multiply/accumulate instructions This includes the
+;; FMA4 floating point multiply/accumulate instructions.  This
-;; scalar version of the instructions as well as the vector
+;; includes the scalar version of the instructions as well as the
+;; vector.
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; In order to match (*a * *b) + *c, particularly when vectorizing, allow
 ;; combine to generate a multiply/add with two memory references.  We then
 ;; add insns, and it can't generate a new pseudo.  I.e.:
 ;;	(set (reg1) (mem (addr1)))
 ;;	(set (reg2) (mult (reg1) (mem (addr2))))
 ;;	(set (reg3) (plus (reg2) (mem (addr3))))
-(define_insn "sse5_fmadd<mode>4"
+(define_insn "fma4_fmadd<mode>4256"
-[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(plus:FMA4MODEF4
+	 (mult:FMA4MODEF4
+	  (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	  (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	 (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x")))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmadd<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+;; Floating multiply and subtract.
+(define_insn "fma4_fmsub<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(minus:FMA4MODEF4
+	 (mult:FMA4MODEF4
+	  (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	  (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	 (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x")))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsub<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+;; Floating point negative multiply and add.
+;; Rewrite (- (a * b) + c) into the canonical form: c - (a * b).
+(define_insn "fma4_fnmadd<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(minus:FMA4MODEF4
+	 (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x")
+	 (mult:FMA4MODEF4
+	  (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	  (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfnmadd<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+;; Floating point negative multiply and subtract.
+(define_insn "fma4_fnmsub<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(minus:FMA4MODEF4
+	 (mult:FMA4MODEF4
+	  (neg:FMA4MODEF4
+	   (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x"))
+	  (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	 (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x")))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfnmsub<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+(define_insn "fma4_fmadd<mode>4"
+[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x")
 	(plus:SSEMODEF4
 	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%0,0,x,xm")
+	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%x,x")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,xm,xm,x"))
+	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,m"))
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x,0,0")))]
+	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x")))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)"
+"vfmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Split fmadd with two memory operands into a load and the fmadd.
-(define_split
-[(set (match_operand:SSEMODEF4 0 "register_operand" "")
-	(plus:SSEMODEF4
-	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" ""))
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "")))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, <MODE>mode);
-emit_insn (gen_sse5_fmadd<mode>4 (operands[0], operands[1],
-				    operands[2], operands[3]));
-DONE;
-})
 ;; For the scalar operations, use operand1 for the upper words that aren't
 ;; modified, so restrict the forms that are generated.
-;; Scalar version of fmadd
+;; Scalar version of fmadd.
-(define_insn "sse5_vmfmadd<mode>4"
+(define_insn "fma4_vmfmadd<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(vec_merge:SSEMODEF2P
 	 (plus:SSEMODEF2P
 	  (mult:SSEMODEF2P
-	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0")
+	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	  (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
-	 (match_dup 1)
+	 (match_dup 0)
 	 (const_int 1)))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
+"vfmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Floating multiply and subtract
+;; Floating multiply and subtract.
-;; Allow two memory operands the same as fmadd
+;; Allow two memory operands the same as fmadd.
-(define_insn "sse5_fmsub<mode>4"
+(define_insn "fma4_fmsub<mode>4"
-[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x")
 	(minus:SSEMODEF4
 	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%0,0,x,xm")
+	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%x,x")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,xm,xm,x"))
+	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,m"))
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x,0,0")))]
+	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x")))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)"
+"vfmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Split fmsub with two memory operands into a load and the fmsub.
-(define_split
-[(set (match_operand:SSEMODEF4 0 "register_operand" "")
-	(minus:SSEMODEF4
-	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" ""))
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "")))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, <MODE>mode);
-emit_insn (gen_sse5_fmsub<mode>4 (operands[0], operands[1],
-				    operands[2], operands[3]));
-DONE;
-})
 ;; For the scalar operations, use operand1 for the upper words that aren't
 ;; modified, so restrict the forms that are generated.
-;; Scalar version of fmsub
+;; Scalar version of fmsub.
-(define_insn "sse5_vmfmsub<mode>4"
+(define_insn "fma4_vmfmsub<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(vec_merge:SSEMODEF2P
 	 (minus:SSEMODEF2P
 	  (mult:SSEMODEF2P
-	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0")
+	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	  (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
-	 (match_dup 1)
+	 (match_dup 0)
 	 (const_int 1)))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"vfmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Floating point negative multiply and add
+;; Floating point negative multiply and add.
-;; Rewrite (- (a * b) + c) into the canonical form: c - (a * b)
+;; Rewrite (- (a * b) + c) into the canonical form: c - (a * b).
-;; Note operands are out of order to simplify call to ix86_sse5_valid_p
+(define_insn "fma4_fnmadd<mode>4"
-;; Allow two memory operands to help in optimizing.
+[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x")
-(define_insn "sse5_fnmadd<mode>4"
-[(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x,x,x")
 	(minus:SSEMODEF4
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x,0,0")
+	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x")
 	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%0,0,x,xm")
+	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%x,x")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,xm,xm,x"))))]
+	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,m"))))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)"
+"vfnmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fnmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Split fnmadd with two memory operands into a load and the fnmadd.
-(define_split
-[(set (match_operand:SSEMODEF4 0 "register_operand" "")
-	(minus:SSEMODEF4
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "")
-	 (mult:SSEMODEF4
-	  (match_operand:SSEMODEF4 1 "nonimmediate_operand" "")
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" ""))))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, true)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, <MODE>mode);
-emit_insn (gen_sse5_fnmadd<mode>4 (operands[0], operands[1],
-				     operands[2], operands[3]));
-DONE;
-})
 ;; For the scalar operations, use operand1 for the upper words that aren't
 ;; modified, so restrict the forms that are generated.
-;; Scalar version of fnmadd
+;; Scalar version of fnmadd.
-(define_insn "sse5_vmfnmadd<mode>4"
+(define_insn "fma4_vmfnmadd<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(vec_merge:SSEMODEF2P
 	 (minus:SSEMODEF2P
 	  (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x")
 	  (mult:SSEMODEF2P
-	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0")
+	   (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm")))
+	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m")))
-	 (match_dup 1)
+	 (match_dup 0)
 	 (const_int 1)))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
+"vfnmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fnmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Floating point negative multiply and subtract
+;; Floating point negative multiply and subtract.
-;; Rewrite (- (a * b) - c) into the canonical form: ((-a) * b) - c
+;; Rewrite (- (a * b) - c) into the canonical form: ((-a) * b) - c.
-;; Allow 2 memory operands to help with optimization
+(define_insn "fma4_fnmsub<mode>4"
-(define_insn "sse5_fnmsub<mode>4"
 [(set (match_operand:SSEMODEF4 0 "register_operand" "=x,x")
 	(minus:SSEMODEF4
 	 (mult:SSEMODEF4
 	  (neg:SSEMODEF4
-	   (match_operand:SSEMODEF4 1 "nonimmediate_operand" "0,0"))
+	   (match_operand:SSEMODEF4 1 "nonimmediate_operand" "%x,x"))
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,xm"))
+	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" "x,m"))
 	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "xm,x")))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, false)"
+"vfnmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fnmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Split fnmsub with two memory operands into a load and the fmsub.
-(define_split
-[(set (match_operand:SSEMODEF4 0 "register_operand" "")
-	(minus:SSEMODEF4
-	 (mult:SSEMODEF4
-	  (neg:SSEMODEF4
-	   (match_operand:SSEMODEF4 1 "nonimmediate_operand" ""))
-	  (match_operand:SSEMODEF4 2 "nonimmediate_operand" ""))
-	 (match_operand:SSEMODEF4 3 "nonimmediate_operand" "")))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, false)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, <MODE>mode);
-emit_insn (gen_sse5_fnmsub<mode>4 (operands[0], operands[1],
-				     operands[2], operands[3]));
-DONE;
-})
 ;; For the scalar operations, use operand1 for the upper words that aren't
 ;; modified, so restrict the forms that are generated.
-;; Scalar version of fnmsub
+;; Scalar version of fnmsub.
-(define_insn "sse5_vmfnmsub<mode>4"
+(define_insn "fma4_vmfnmsub<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(vec_merge:SSEMODEF2P
 	 (minus:SSEMODEF2P
 	  (mult:SSEMODEF2P
 	   (neg:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0"))
+	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x"))
-	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	   (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	  (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
-	 (match_dup 1)
+	 (match_dup 0)
 	 (const_int 1)))]
-"TARGET_SSE5 && TARGET_FUSED_MADD
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-&& ix86_sse5_valid_op_p (operands, insn, 4, true, 2, false)"
+"vfnmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fnmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; The same instructions using an UNSPEC to allow the intrinsic to be used
+(define_insn "fma4i_fmadd<mode>4256"
-;; even if the user used -mno-fused-madd
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
-;; Parallel instructions.  During instruction generation, just default
+	(unspec:FMA4MODEF4
-;; to registers, and let combine later build the appropriate instruction.
+	 [(plus:FMA4MODEF4
-(define_expand "sse5i_fmadd<mode>4"
+	   (mult:FMA4MODEF4
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
+	    (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	    (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	   (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x"))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmadd<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+(define_insn "fma4i_fmsub<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(unspec:FMA4MODEF4
+	 [(minus:FMA4MODEF4
+	   (mult:FMA4MODEF4
+	    (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	    (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	   (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x"))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsub<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+(define_insn "fma4i_fnmadd<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(unspec:FMA4MODEF4
+	 [(minus:FMA4MODEF4
+	   (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x")
+	   (mult:FMA4MODEF4
+	    (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x")
+	    (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m")))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfnmadd<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+(define_insn "fma4i_fnmsub<mode>4256"
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x,x")
+	(unspec:FMA4MODEF4
+	 [(minus:FMA4MODEF4
+	   (mult:FMA4MODEF4
+	    (neg:FMA4MODEF4
+	     (match_operand:FMA4MODEF4 1 "nonimmediate_operand" "%x,x"))
+	    (match_operand:FMA4MODEF4 2 "nonimmediate_operand" "x,m"))
+	   (match_operand:FMA4MODEF4 3 "nonimmediate_operand" "xm,x"))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfnmsub<fma4modesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "<MODE>")])
+(define_insn "fma4i_fmadd<mode>4"
+[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(plus:SSEMODEF2P
 	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "register_operand" "")
+	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	    (match_operand:SSEMODEF2P 2 "register_operand" ""))
+	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
-	   (match_operand:SSEMODEF2P 3 "register_operand" ""))]
+	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-{
+"vfmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_fmadd<mode>4 (operands[0], operands[1],
-					operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_fmadd<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x,x,x")
-	(unspec:SSEMODEF2P
-	 [(plus:SSEMODEF2P
-	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%0,0,x,xm")
-	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm,xm,x"))
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x,0,0"))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
-"fmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-(define_expand "sse5i_fmsub<mode>4"
+(define_insn "fma4i_fmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
+[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(minus:SSEMODEF2P
 	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "register_operand" "")
+	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	    (match_operand:SSEMODEF2P 2 "register_operand" ""))
+	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
-	   (match_operand:SSEMODEF2P 3 "register_operand" ""))]
+	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-{
+"vfmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
+[(set_attr "type" "ssemuladd")
-if (TARGET_FUSED_MADD)
+(set_attr "mode" "<MODE>")])
-{
-emit_insn (gen_sse5_fmsub<mode>4 (operands[0], operands[1],
+(define_insn "fma4i_fnmadd<mode>4"
-					operands[2], operands[3]));
+[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
-DONE;
-}
-})
-(define_insn "*sse5i_fmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x,x,x")
 	(unspec:SSEMODEF2P
 	 [(minus:SSEMODEF2P
+	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x")
 	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%0,0,x,xm")
+	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm,xm,x"))
+	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m")))]
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x,0,0"))]
+	 UNSPEC_FMA4_INTRINSIC))]
-	 UNSPEC_SSE5_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
+"vfnmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-"fmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Rewrite (- (a * b) + c) into the canonical form: c - (a * b)
+(define_insn "fma4i_fnmsub<mode>4"
-;; Note operands are out of order to simplify call to ix86_sse5_valid_p
+[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
-(define_expand "sse5i_fnmadd<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(unspec:SSEMODEF2P
-	 [(minus:SSEMODEF2P
-	   (match_operand:SSEMODEF2P 3 "register_operand" "")
-	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "register_operand" "")
-	    (match_operand:SSEMODEF2P 2 "register_operand" "")))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5"
-{
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_fnmadd<mode>4 (operands[0], operands[1],
-					 operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_fnmadd<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x,x,x")
-	(unspec:SSEMODEF2P
-	 [(minus:SSEMODEF2P
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x,0,0")
-	   (mult:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%0,0,x,xm")
-	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm,xm,x")))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
-"fnmadd<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-[(set_attr "type" "ssemuladd")
-(set_attr "mode" "<MODE>")])
-;; Rewrite (- (a * b) - c) into the canonical form: ((-a) * b) - c
-(define_expand "sse5i_fnmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
 	(unspec:SSEMODEF2P
 	 [(minus:SSEMODEF2P
 	   (mult:SSEMODEF2P
 	    (neg:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" ""))
+	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x"))
-	    (match_operand:SSEMODEF2P 2 "register_operand" ""))
+	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
-	   (match_operand:SSEMODEF2P 3 "register_operand" ""))]
+	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-{
+"vfnmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_fnmsub<mode>4 (operands[0], operands[1],
-					 operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_fnmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x,x,x")
-	(unspec:SSEMODEF2P
-	 [(minus:SSEMODEF2P
-	   (mult:SSEMODEF2P
-	    (neg:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0,x,xm"))
-	    (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm,xm,x"))
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x,0,0"))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
-"fnmsub<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<MODE>")])
-;; Scalar instructions
-(define_expand "sse5i_vmfmadd<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(unspec:SSEMODEF2P
-	 [(vec_merge:SSEMODEF2P
-	   (plus:SSEMODEF2P
-	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" "")
-	     (match_operand:SSEMODEF2P 2 "register_operand" ""))
-	    (match_operand:SSEMODEF2P 3 "register_operand" ""))
-	   (match_dup 1)
-	   (const_int 0))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5"
-{
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_vmfmadd<mode>4 (operands[0], operands[1],
-					  operands[2], operands[3]));
-DONE;
-}
-})
 ;; For the scalar operations, use operand1 for the upper words that aren't
 ;; modified, so restrict the forms that are accepted.
-(define_insn "*sse5i_vmfmadd<mode>4"
+(define_insn "fma4i_vmfmadd<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(vec_merge:SSEMODEF2P
 	   (plus:SSEMODEF2P
 	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" "0,0")
+	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	    (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
 	   (match_dup 0)
-	   (const_int 0))]
+	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-"fmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vfmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<ssescalarmode>")])
-(define_expand "sse5i_vmfmsub<mode>4"
+(define_insn "fma4i_vmfmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(unspec:SSEMODEF2P
-	 [(vec_merge:SSEMODEF2P
-	   (minus:SSEMODEF2P
-	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" "")
-	     (match_operand:SSEMODEF2P 2 "register_operand" ""))
-	    (match_operand:SSEMODEF2P 3 "register_operand" ""))
-	   (match_dup 0)
-	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5"
-{
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_vmfmsub<mode>4 (operands[0], operands[1],
-					  operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_vmfmsub<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(vec_merge:SSEMODEF2P
 	   (minus:SSEMODEF2P
 	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" "0,0")
+	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	    (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
-	   (match_dup 1)
+	   (match_dup 0)
 	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-"fmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vfmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<ssescalarmode>")])
-;; Note operands are out of order to simplify call to ix86_sse5_valid_p
+(define_insn "fma4i_vmfnmadd<mode>4"
-(define_expand "sse5i_vmfnmadd<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(unspec:SSEMODEF2P
-	 [(vec_merge:SSEMODEF2P
-	   (minus:SSEMODEF2P
-	    (match_operand:SSEMODEF2P 3 "register_operand" "")
-	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "register_operand" "")
-	     (match_operand:SSEMODEF2P 2 "register_operand" "")))
-	   (match_dup 1)
-	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5"
-{
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_vmfnmadd<mode>4 (operands[0], operands[1],
-					   operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_vmfnmadd<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(vec_merge:SSEMODEF2P
 	   (minus:SSEMODEF2P
 	    (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x")
 	    (mult:SSEMODEF2P
-	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%0,0")
+	     (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x")
-	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm")))
+	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m")))
-	   (match_dup 1)
+	   (match_dup 0)
 	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, true)"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-"fnmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vfnmadd<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<ssescalarmode>")])
-(define_expand "sse5i_vmfnmsub<mode>4"
+(define_insn "fma4i_vmfnmsub<mode>4"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(unspec:SSEMODEF2P
-	 [(vec_merge:SSEMODEF2P
-	   (minus:SSEMODEF2P
-	    (mult:SSEMODEF2P
-	     (neg:SSEMODEF2P
-	      (match_operand:SSEMODEF2P 1 "register_operand" ""))
-	     (match_operand:SSEMODEF2P 2 "register_operand" ""))
-	    (match_operand:SSEMODEF2P 3 "register_operand" ""))
-	   (match_dup 1)
-	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
-"TARGET_SSE5"
-{
-/* If we have -mfused-madd, emit the normal insn rather than the UNSPEC */
-if (TARGET_FUSED_MADD)
-{
-emit_insn (gen_sse5_vmfnmsub<mode>4 (operands[0], operands[1],
-					   operands[2], operands[3]));
-DONE;
-}
-})
-(define_insn "*sse5i_vmfnmsub<mode>4"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x")
 	(unspec:SSEMODEF2P
 	 [(vec_merge:SSEMODEF2P
 	   (minus:SSEMODEF2P
 	    (mult:SSEMODEF2P
 	     (neg:SSEMODEF2P
-	      (match_operand:SSEMODEF2P 1 "register_operand" "0,0"))
+	      (match_operand:SSEMODEF2P 1 "nonimmediate_operand" "%x,x"))
-	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm"))
+	     (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,m"))
 	    (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm,x"))
-	   (match_dup 1)
+	   (match_dup 0)
 	   (const_int 1))]
-	 UNSPEC_SSE5_INTRINSIC))]
+	 UNSPEC_FMA4_INTRINSIC))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_FMA4 && TARGET_FUSED_MADD"
-"fnmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vfnmsub<ssemodesuffixf2s>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "<ssescalarmode>")])
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; FMA4 Parallel floating point multiply addsub and subadd operations.
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(define_insn "fma4_fmaddsubv8sf4"
+[(set (match_operand:V8SF 0 "register_operand" "=x,x")
+	(vec_merge:V8SF
+	  (plus:V8SF
+	    (mult:V8SF
+	      (match_operand:V8SF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V8SF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V8SF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V8SF
+	    (mult:V8SF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 170)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V8SF")])
+(define_insn "fma4_fmaddsubv4df4"
+[(set (match_operand:V4DF 0 "register_operand" "=x,x")
+	(vec_merge:V4DF
+	  (plus:V4DF
+	    (mult:V4DF
+	      (match_operand:V4DF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V4DF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V4DF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V4DF
+	    (mult:V4DF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 10)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4DF")])
+(define_insn "fma4_fmaddsubv4sf4"
+[(set (match_operand:V4SF 0 "register_operand" "=x,x")
+	(vec_merge:V4SF
+	  (plus:V4SF
+	    (mult:V4SF
+	      (match_operand:V4SF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V4SF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V4SF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V4SF
+	    (mult:V4SF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 10)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4SF")])
+(define_insn "fma4_fmaddsubv2df4"
+[(set (match_operand:V2DF 0 "register_operand" "=x,x")
+	(vec_merge:V2DF
+	  (plus:V2DF
+	    (mult:V2DF
+	      (match_operand:V2DF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V2DF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V2DF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V2DF
+	    (mult:V2DF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 2)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V2DF")])
+(define_insn "fma4_fmsubaddv8sf4"
+[(set (match_operand:V8SF 0 "register_operand" "=x,x")
+	(vec_merge:V8SF
+	  (plus:V8SF
+	    (mult:V8SF
+	      (match_operand:V8SF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V8SF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V8SF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V8SF
+	    (mult:V8SF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 85)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V8SF")])
+(define_insn "fma4_fmsubaddv4df4"
+[(set (match_operand:V4DF 0 "register_operand" "=x,x")
+	(vec_merge:V4DF
+	  (plus:V4DF
+	    (mult:V4DF
+	      (match_operand:V4DF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V4DF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V4DF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V4DF
+	    (mult:V4DF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 5)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4DF")])
+(define_insn "fma4_fmsubaddv4sf4"
+[(set (match_operand:V4SF 0 "register_operand" "=x,x")
+	(vec_merge:V4SF
+	  (plus:V4SF
+	    (mult:V4SF
+	      (match_operand:V4SF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V4SF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V4SF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V4SF
+	    (mult:V4SF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 5)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4SF")])
+(define_insn "fma4_fmsubaddv2df4"
+[(set (match_operand:V2DF 0 "register_operand" "=x,x")
+	(vec_merge:V2DF
+	  (plus:V2DF
+	    (mult:V2DF
+	      (match_operand:V2DF 1 "nonimmediate_operand" "%x,x")
+	      (match_operand:V2DF 2 "nonimmediate_operand" "x,m"))
+	    (match_operand:V2DF 3 "nonimmediate_operand" "xm,x"))
+	  (minus:V2DF
+	    (mult:V2DF
+	      (match_dup 1)
+	      (match_dup 2))
+	    (match_dup 3))
+	  (const_int 1)))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V2DF")])
+(define_insn "fma4i_fmaddsubv8sf4"
+[(set (match_operand:V8SF 0 "register_operand" "=x,x")
+	(unspec:V8SF
+	 [(vec_merge:V8SF
+	   (plus:V8SF
+	     (mult:V8SF
+	       (match_operand:V8SF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V8SF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V8SF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V8SF
+	     (mult:V8SF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 170))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V8SF")])
+(define_insn "fma4i_fmaddsubv4df4"
+[(set (match_operand:V4DF 0 "register_operand" "=x,x")
+	(unspec:V4DF
+	 [(vec_merge:V4DF
+	   (plus:V4DF
+	     (mult:V4DF
+	       (match_operand:V4DF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V4DF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V4DF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V4DF
+	     (mult:V4DF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 10))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4DF")])
+(define_insn "fma4i_fmaddsubv4sf4"
+[(set (match_operand:V4SF 0 "register_operand" "=x,x")
+	(unspec:V4SF
+	 [(vec_merge:V4SF
+	   (plus:V4SF
+	     (mult:V4SF
+	       (match_operand:V4SF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V4SF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V4SF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V4SF
+	     (mult:V4SF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 10))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4SF")])
+(define_insn "fma4i_fmaddsubv2df4"
+[(set (match_operand:V2DF 0 "register_operand" "=x,x")
+	(unspec:V2DF
+	 [(vec_merge:V2DF
+	   (plus:V2DF
+	     (mult:V2DF
+	       (match_operand:V2DF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V2DF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V2DF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V2DF
+	     (mult:V2DF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 2))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmaddsubpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V2DF")])
+(define_insn "fma4i_fmsubaddv8sf4"
+[(set (match_operand:V8SF 0 "register_operand" "=x,x")
+	(unspec:V8SF
+	 [(vec_merge:V8SF
+	   (plus:V8SF
+	     (mult:V8SF
+	       (match_operand:V8SF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V8SF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V8SF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V8SF
+	     (mult:V8SF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 85))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V8SF")])
+(define_insn "fma4i_fmsubaddv4df4"
+[(set (match_operand:V4DF 0 "register_operand" "=x,x")
+	(unspec:V4DF
+	 [(vec_merge:V4DF
+	   (plus:V4DF
+	     (mult:V4DF
+	       (match_operand:V4DF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V4DF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V4DF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V4DF
+	     (mult:V4DF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 5))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4DF")])
+(define_insn "fma4i_fmsubaddv4sf4"
+[(set (match_operand:V4SF 0 "register_operand" "=x,x")
+	(unspec:V4SF
+	 [(vec_merge:V4SF
+	   (plus:V4SF
+	     (mult:V4SF
+	       (match_operand:V4SF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V4SF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V4SF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V4SF
+	     (mult:V4SF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 5))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V4SF")])
+(define_insn "fma4i_fmsubaddv2df4"
+[(set (match_operand:V2DF 0 "register_operand" "=x,x")
+	(unspec:V2DF
+	 [(vec_merge:V2DF
+	   (plus:V2DF
+	     (mult:V2DF
+	       (match_operand:V2DF 1 "nonimmediate_operand" "%x,x")
+	       (match_operand:V2DF 2 "nonimmediate_operand" "x,m"))
+	     (match_operand:V2DF 3 "nonimmediate_operand" "xm,x"))
+	   (minus:V2DF
+	     (mult:V2DF
+	       (match_dup 1)
+	       (match_dup 2))
+	     (match_dup 3))
+	   (const_int 1))]
+	 UNSPEC_FMA4_INTRINSIC))]
+"TARGET_FMA4 && TARGET_FUSED_MADD"
+"vfmsubaddpd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "ssemuladd")
+(set_attr "mode" "V2DF")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Parallel single-precision floating point conversion operations
 ;;
 	  (parallel [(const_int 0) (const_int 1)])))]
 "TARGET_SSE"
 "cvttps2pi\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
 (set_attr "unit" "mmx")
+(set_attr "prefix_rep" "0")
 (set_attr "mode" "SF")])
 (define_insn "*avx_cvtsi2ss"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_merge:V4SF
 	  (match_operand:V4SF 1 "register_operand" "x")
 	  (const_int 1)))]
 "TARGET_AVX && TARGET_64BIT"
 "vcvtsi2ssq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseicvt")
+(set_attr "length_vex" "4")
 (set_attr "prefix" "vex")
 (set_attr "mode" "SF")])
 (define_insn "sse_cvtsi2ssq"
 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
 	  (match_operand:V4SF 1 "register_operand" "0,0")
 	  (const_int 1)))]
 "TARGET_SSE && TARGET_64BIT"
 "cvtsi2ssq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseicvt")
+(set_attr "prefix_rex" "1")
 (set_attr "athlon_decode" "vector,double")
 (set_attr "amdfam10_decode" "vector,double")
 (set_attr "mode" "SF")])
 (define_insn "sse_cvtss2si"
 "TARGET_SSE2"
 "cvtdq2ps\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
 (set_attr "mode" "V4SF")])
+(define_expand "sse2_cvtudq2ps"
+[(set (match_dup 5)
+	(float:V4SF (match_operand:V4SI 1 "nonimmediate_operand" "")))
+(set (match_dup 6)
+	(lt:V4SF (match_dup 5) (match_dup 3)))
+(set (match_dup 7)
+	(and:V4SF (match_dup 6) (match_dup 4)))
+(set (match_operand:V4SF 0 "register_operand" "")
+	(plus:V4SF (match_dup 5) (match_dup 7)))]
+"TARGET_SSE2"
+{
+REAL_VALUE_TYPE TWO32r;
+rtx x;
+int i;
+real_ldexp (&TWO32r, &dconst1, 32);
+x = const_double_from_real_value (TWO32r, SFmode);
+operands[3] = force_reg (V4SFmode, CONST0_RTX (V4SFmode));
+operands[4] = force_reg (V4SFmode, ix86_build_const_vector (SFmode, 1, x));
+for (i = 5; i < 8; i++)
+operands[i] = gen_reg_rtx (V4SFmode);
+})
 (define_insn "avx_cvtps2dq<avxmodesuffix>"
 [(set (match_operand:AVXMODEDCVTPS2DQ 0 "register_operand" "=x")
 	(unspec:AVXMODEDCVTPS2DQ
 	  [(match_operand:<avxcvtvecmode> 1 "nonimmediate_operand" "xm")]
 	  UNSPEC_FIX_NOTRUNC))]
 	(fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
 "TARGET_SSE2"
 "cvttps2dq\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
 (set_attr "prefix_rep" "1")
+(set_attr "prefix_data16" "0")
 (set_attr "mode" "TI")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Parallel double-precision floating point conversion operations
 	(float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "y,m")))]
 "TARGET_SSE2"
 "cvtpi2pd\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
 (set_attr "unit" "mmx,*")
+(set_attr "prefix_data16" "1,*")
 (set_attr "mode" "V2DF")])
 (define_insn "sse2_cvtpd2pi"
 [(set (match_operand:V2SI 0 "register_operand" "=y")
 	(unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")]
 	  (match_operand:V2DF 1 "register_operand" "x")
 	  (const_int 1)))]
 "TARGET_AVX && TARGET_64BIT"
 "vcvtsi2sdq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseicvt")
+(set_attr "length_vex" "4")
 (set_attr "prefix" "vex")
 (set_attr "mode" "DF")])
 (define_insn "sse2_cvtsi2sdq"
 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
 	  (match_operand:V2DF 1 "register_operand" "0,0")
 	  (const_int 1)))]
 "TARGET_SSE2 && TARGET_64BIT"
 "cvtsi2sdq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseicvt")
+(set_attr "prefix_rex" "1")
 (set_attr "mode" "DF")
 (set_attr "athlon_decode" "double,direct")
 (set_attr "amdfam10_decode" "vector,double")])
 (define_insn "sse2_cvtsd2si"
 "TARGET_SSE2"
 "* return TARGET_AVX ? \"vcvtpd2dq{x}\t{%1, %0|%0, %1}\"
 		       : \"cvtpd2dq\t{%1, %0|%0, %1}\";"
 [(set_attr "type" "ssecvt")
 (set_attr "prefix_rep" "1")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")
 (set_attr "amdfam10_decode" "double")])
 (define_insn "avx_cvttpd2dq256"
 	  (match_operand:V2SI 2 "const0_operand" "")))]
 "TARGET_SSE2"
 "* return TARGET_AVX ? \"vcvttpd2dq{x}\t{%1, %0|%0, %1}\"
 		       : \"cvttpd2dq\t{%1, %0|%0, %1}\";"
 [(set_attr "type" "ssecvt")
-(set_attr "prefix_rep" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")
 (set_attr "amdfam10_decode" "double")])
 (define_insn "*avx_cvtsd2ss"
 "TARGET_SSE2"
 "%vcvtps2pd\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "V2DF")
+(set_attr "prefix_data16" "0")
 (set_attr "amdfam10_decode" "direct")])
 (define_expand "vec_unpacks_hi_v4sf"
 [(set (match_dup 2)
 (vec_select:V4SF
 (float:V2DF
 	  (vec_select:V2SI
 	  (match_dup 2)
 	    (parallel [(const_int 0) (const_int 1)]))))]
 "TARGET_SSE2"
-{
+"operands[2] = gen_reg_rtx (V4SImode);")
-operands[2] = gen_reg_rtx (V4SImode);
-})
 (define_expand "vec_unpacks_float_lo_v4si"
 [(set (match_operand:V2DF 0 "register_operand" "")
 	(float:V2DF
 	  (vec_select:V2SI
 	    (match_operand:V4SI 1 "nonimmediate_operand" "")
 	    (parallel [(const_int 0) (const_int 1)]))))]
 "TARGET_SSE2")
+(define_expand "vec_unpacku_float_hi_v4si"
+[(set (match_dup 5)
+	(vec_select:V4SI
+	  (match_operand:V4SI 1 "nonimmediate_operand" "")
+	  (parallel [(const_int 2)
+		     (const_int 3)
+		     (const_int 2)
+		     (const_int 3)])))
+(set (match_dup 6)
+(float:V2DF
+	  (vec_select:V2SI
+	  (match_dup 5)
+	    (parallel [(const_int 0) (const_int 1)]))))
+(set (match_dup 7)
+	(lt:V2DF (match_dup 6) (match_dup 3)))
+(set (match_dup 8)
+	(and:V2DF (match_dup 7) (match_dup 4)))
+(set (match_operand:V2DF 0 "register_operand" "")
+	(plus:V2DF (match_dup 6) (match_dup 8)))]
+"TARGET_SSE2"
+{
+REAL_VALUE_TYPE TWO32r;
+rtx x;
+int i;
+real_ldexp (&TWO32r, &dconst1, 32);
+x = const_double_from_real_value (TWO32r, DFmode);
+operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
+operands[4] = force_reg (V2DFmode, ix86_build_const_vector (DFmode, 1, x));
+operands[5] = gen_reg_rtx (V4SImode);
+for (i = 6; i < 9; i++)
+operands[i] = gen_reg_rtx (V2DFmode);
+})
+(define_expand "vec_unpacku_float_lo_v4si"
+[(set (match_dup 5)
+	(float:V2DF
+	  (vec_select:V2SI
+	    (match_operand:V4SI 1 "nonimmediate_operand" "")
+	    (parallel [(const_int 0) (const_int 1)]))))
+(set (match_dup 6)
+	(lt:V2DF (match_dup 5) (match_dup 3)))
+(set (match_dup 7)
+	(and:V2DF (match_dup 6) (match_dup 4)))
+(set (match_operand:V2DF 0 "register_operand" "")
+	(plus:V2DF (match_dup 5) (match_dup 7)))]
+"TARGET_SSE2"
+{
+REAL_VALUE_TYPE TWO32r;
+rtx x;
+int i;
+real_ldexp (&TWO32r, &dconst1, 32);
+x = const_double_from_real_value (TWO32r, DFmode);
+operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
+operands[4] = force_reg (V2DFmode, ix86_build_const_vector (DFmode, 1, x));
+for (i = 5; i < 8; i++)
+operands[i] = gen_reg_rtx (V2DFmode);
+})
 (define_expand "vec_pack_trunc_v2df"
 [(match_operand:V4SF 0 "register_operand" "")
 (match_operand:V2DF 1 "nonimmediate_operand" "")
 (match_operand:V2DF 2 "nonimmediate_operand" "")]
 "TARGET_SSE2"
 r1 = gen_reg_rtx (V4SImode);
 r2 = gen_reg_rtx (V4SImode);
 emit_insn (gen_sse2_cvttpd2dq (r1, operands[1]));
 emit_insn (gen_sse2_cvttpd2dq (r2, operands[2]));
-emit_insn (gen_sse2_punpcklqdq (gen_lowpart (V2DImode, operands[0]),
+emit_insn (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
-				  gen_lowpart (V2DImode, r1),
+	    				 gen_lowpart (V2DImode, r1),
-				  gen_lowpart (V2DImode, r2)));
+					 gen_lowpart (V2DImode, r2)));
 DONE;
 })
 (define_expand "vec_pack_sfix_v2df"
 [(match_operand:V4SI 0 "register_operand" "")
 r1 = gen_reg_rtx (V4SImode);
 r2 = gen_reg_rtx (V4SImode);
 emit_insn (gen_sse2_cvtpd2dq (r1, operands[1]));
 emit_insn (gen_sse2_cvtpd2dq (r2, operands[2]));
-emit_insn (gen_sse2_punpcklqdq (gen_lowpart (V2DImode, operands[0]),
+emit_insn (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
-				  gen_lowpart (V2DImode, r1),
+	    				 gen_lowpart (V2DImode, r1),
-				  gen_lowpart (V2DImode, r2)));
+					 gen_lowpart (V2DImode, r2)));
 DONE;
 })
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 movhps\t{%2, %0|%0, %2}
 movlps\t{%2, %H0|%H0, %2}"
 [(set_attr "type" "ssemov")
 (set_attr "mode" "V4SF,V2SF,V2SF")])
+;; Recall that the 256-bit unpck insns only shuffle within their lanes.
 (define_insn "avx_unpckhps256"
 [(set (match_operand:V8SF 0 "register_operand" "=x")
 	(vec_select:V8SF
 	  (vec_concat:V16SF
 	    (match_operand:V8SF 1 "register_operand" "x")
 "vunpckhps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
-(define_insn "*avx_unpckhps"
+(define_insn "*avx_interleave_highv4sf"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_select:V4SF
 	  (vec_concat:V8SF
 	    (match_operand:V4SF 1 "register_operand" "x")
 	    (match_operand:V4SF 2 "nonimmediate_operand" "xm"))
 "vunpckhps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF")])
-(define_insn "sse_unpckhps"
+(define_insn "vec_interleave_highv4sf"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_select:V4SF
 	  (vec_concat:V8SF
 	    (match_operand:V4SF 1 "register_operand" "0")
 	    (match_operand:V4SF 2 "nonimmediate_operand" "xm"))
 "TARGET_SSE"
 "unpckhps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "mode" "V4SF")])
+;; Recall that the 256-bit unpck insns only shuffle within their lanes.
 (define_insn "avx_unpcklps256"
 [(set (match_operand:V8SF 0 "register_operand" "=x")
 	(vec_select:V8SF
 	  (vec_concat:V16SF
 	    (match_operand:V8SF 1 "register_operand" "x")
 "vunpcklps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
-(define_insn "*avx_unpcklps"
+(define_insn "*avx_interleave_lowv4sf"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_select:V4SF
 	  (vec_concat:V8SF
 	    (match_operand:V4SF 1 "register_operand" "x")
 	    (match_operand:V4SF 2 "nonimmediate_operand" "xm"))
 "vunpcklps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF")])
-(define_insn "sse_unpcklps"
+(define_insn "vec_interleave_lowv4sf"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_select:V4SF
 	  (vec_concat:V8SF
 	    (match_operand:V4SF 1 "register_operand" "0")
 	    (match_operand:V4SF 2 "nonimmediate_operand" "xm"))
 operands[3] = GEN_INT (mask);
 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_expand "sse_shufps"
 [(match_operand:V4SF 0 "register_operand" "")
 operands[3] = GEN_INT (mask);
 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF")])
 (define_insn "sse_shufps_<mode>"
 [(set (match_operand:SSEMODE4S 0 "register_operand" "=x")
 operands[3] = GEN_INT (mask);
 return "shufps\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V4SF")])
 (define_insn "sse_storehps"
 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,x,x")
 	(vec_select:V2SF
 "@
 shufps\t{$0xe4, %1, %2, %0|%0, %2, %1, 0xe4}
 vmovlps\t{%2, %1, %0|%0, %1, %2}
 vmovlps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog,ssemov,ssemov")
+(set_attr "length_immediate" "1,*,*")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF,V2SF,V2SF")])
 (define_insn "sse_loadlps"
 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,m")
 "@
 shufps\t{$0xe4, %1, %0|%0, %1, 0xe4}
 movlps\t{%2, %0|%0, %2}
 movlps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog,ssemov,ssemov")
+(set_attr "length_immediate" "1,*,*")
 (set_attr "mode" "V4SF,V2SF,V2SF")])
 (define_insn "*avx_movss"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_merge:V4SF
 "TARGET_SSE"
 "movss\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssemov")
 (set_attr "mode" "SF")])
+(define_expand "vec_dupv4sf"
+[(set (match_operand:V4SF 0 "register_operand" "")
+	(vec_duplicate:V4SF
+	  (match_operand:SF 1 "nonimmediate_operand" "")))]
+"TARGET_SSE"
+{
+if (!TARGET_AVX)
+operands[1] = force_reg (V4SFmode, operands[1]);
+})
 (define_insn "*vec_dupv4sf_avx"
-[(set (match_operand:V4SF 0 "register_operand" "=x")
+[(set (match_operand:V4SF 0 "register_operand" "=x,x")
 	(vec_duplicate:V4SF
-	  (match_operand:SF 1 "register_operand" "x")))]
+	  (match_operand:SF 1 "nonimmediate_operand" "x,m")))]
 "TARGET_AVX"
-"vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}"
+"@
-[(set_attr "type" "sselog1")
+vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}
+vbroadcastss\t{%1, %0|%0, %1}"
+[(set_attr "type" "sselog1,ssemov")
+(set_attr "length_immediate" "1,0")
+(set_attr "prefix_extra" "0,1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF")])
 (define_insn "*vec_dupv4sf"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(vec_duplicate:V4SF
 	  (match_operand:SF 1 "register_operand" "0")))]
 "TARGET_SSE"
 "shufps\t{$0, %0, %0|%0, %0, 0}"
 [(set_attr "type" "sselog1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V4SF")])
 (define_insn "*vec_concatv2sf_avx"
 [(set (match_operand:V2SF 0 "register_operand"     "=x,x,x,*y ,*y")
 	(vec_concat:V2SF
 vinsertps\t{$0x10, %2, %1, %0|%0, %1, %2, 0x10}
 vmovss\t{%1, %0|%0, %1}
 punpckldq\t{%2, %0|%0, %2}
 movd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog,sselog,ssemov,mmxcvt,mmxmov")
+(set_attr "length_immediate" "*,1,*,*,*")
+(set_attr "prefix_extra" "*,1,*,*,*")
 (set (attr "prefix")
 (if_then_else (eq_attr "alternative" "3,4")
 (const_string "orig")
 (const_string "vex")))
 (set_attr "mode" "V4SF,V4SF,SF,DI,DI")])
 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
 movss\t{%1, %0|%0, %1}
 punpckldq\t{%2, %0|%0, %2}
 movd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog,sselog,ssemov,mmxcvt,mmxmov")
+(set_attr "prefix_data16" "*,1,*,*,*")
 (set_attr "prefix_extra" "*,1,*,*,*")
+(set_attr "length_immediate" "*,1,*,*,*")
 (set_attr "mode" "V4SF,V4SF,SF,DI,DI")])
 ;; ??? In theory we can match memory for the MMX alternative, but allowing
 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
 ;; alternatives pretty much forces the MMX alternative to be chosen.
 {
 ix86_expand_vector_init (false, operands[0], operands[1]);
 DONE;
 })
-(define_insn "*vec_setv4sf_0_avx"
+(define_insn "*vec_set<mode>_0_avx"
-[(set (match_operand:V4SF 0 "nonimmediate_operand"  "=x,x,x,m")
+[(set (match_operand:SSEMODE4S 0 "nonimmediate_operand"  "=x,x, x,x,  x,m")
-	(vec_merge:V4SF
+	(vec_merge:SSEMODE4S
-	  (vec_duplicate:V4SF
+	  (vec_duplicate:SSEMODE4S
-	    (match_operand:SF 2 "general_operand"     " x,m,*r,x*rfF"))
+	    (match_operand:<ssescalarmode> 2
-	  (match_operand:V4SF 1 "vector_move_operand" " x,C,C ,0")
+	      "general_operand"                            " x,m,*r,x,*rm,x*rfF"))
+	  (match_operand:SSEMODE4S 1 "vector_move_operand" " C,C, C,x,  x,0")
 	  (const_int 1)))]
 "TARGET_AVX"
 "@
+vinsertps\t{$0xe, %2, %2, %0|%0, %2, %2, 0xe}
+vmov<ssescalarmodesuffix2s>\t{%2, %0|%0, %2}
+vmovd\t{%2, %0|%0, %2}
 vmovss\t{%2, %1, %0|%0, %1, %2}
-vmovss\t{%2, %0|%0, %2}
+vpinsrd\t{$0, %2, %1, %0|%0, %1, %2, 0}
-vmovd\t{%2, %0|%0, %2}
+#"
+[(set_attr "type" "sselog,ssemov,ssemov,ssemov,sselog,*")
+(set_attr "prefix_extra" "*,*,*,*,1,*")
+(set_attr "length_immediate" "*,*,*,*,1,*")
+(set_attr "prefix" "vex")
+(set_attr "mode" "SF,<ssescalarmode>,SI,SF,TI,*")])
+(define_insn "*vec_set<mode>_0_sse4_1"
+[(set (match_operand:SSEMODE4S 0 "nonimmediate_operand"  "=x,x, x,x,  x,m")
+	(vec_merge:SSEMODE4S
+	  (vec_duplicate:SSEMODE4S
+	    (match_operand:<ssescalarmode> 2
+	      "general_operand"                            " x,m,*r,x,*rm,*rfF"))
+	  (match_operand:SSEMODE4S 1 "vector_move_operand" " C,C, C,0,  0,0")
+	  (const_int 1)))]
+"TARGET_SSE4_1"
+"@
+insertps\t{$0xe, %2, %0|%0, %2, 0xe}
+mov<ssescalarmodesuffix2s>\t{%2, %0|%0, %2}
+movd\t{%2, %0|%0, %2}
+movss\t{%2, %0|%0, %2}
+pinsrd\t{$0, %2, %0|%0, %2, 0}
+#"
+[(set_attr "type" "sselog,ssemov,ssemov,ssemov,sselog,*")
+(set_attr "prefix_extra" "*,*,*,*,1,*")
+(set_attr "length_immediate" "*,*,*,*,1,*")
+(set_attr "mode" "SF,<ssescalarmode>,SI,SF,TI,*")])
+(define_insn "*vec_set<mode>_0_sse2"
+[(set (match_operand:SSEMODE4S 0 "nonimmediate_operand"  "=x, x,x,m")
+	(vec_merge:SSEMODE4S
+	  (vec_duplicate:SSEMODE4S
+	    (match_operand:<ssescalarmode> 2
+	      "general_operand"                            " m,*r,x,x*rfF"))
+	  (match_operand:SSEMODE4S 1 "vector_move_operand" " C, C,0,0")
+	  (const_int 1)))]
+"TARGET_SSE2"
+"@
+mov<ssescalarmodesuffix2s>\t{%2, %0|%0, %2}
+movd\t{%2, %0|%0, %2}
+movss\t{%2, %0|%0, %2}
 #"
 [(set_attr "type" "ssemov")
-(set_attr "prefix" "vex")
+(set_attr "mode" "<ssescalarmode>,SI,SF,*")])
-(set_attr "mode" "SF")])
+(define_insn "vec_set<mode>_0"
-(define_insn "vec_setv4sf_0"
+[(set (match_operand:SSEMODE4S 0 "nonimmediate_operand"  "=x,x,m")
-[(set (match_operand:V4SF 0 "nonimmediate_operand"  "=x,x,Y2,m")
+	(vec_merge:SSEMODE4S
-	(vec_merge:V4SF
+	  (vec_duplicate:SSEMODE4S
-	  (vec_duplicate:V4SF
+	    (match_operand:<ssescalarmode> 2
-	    (match_operand:SF 2 "general_operand"     " x,m,*r,x*rfF"))
+	      "general_operand"                            " m,x,x*rfF"))
-	  (match_operand:V4SF 1 "vector_move_operand" " 0,C,C ,0")
+	  (match_operand:SSEMODE4S 1 "vector_move_operand" " C,0,0")
 	  (const_int 1)))]
 "TARGET_SSE"
 "@
 movss\t{%2, %0|%0, %2}
 movss\t{%2, %0|%0, %2}
-movd\t{%2, %0|%0, %2}
 #"
 [(set_attr "type" "ssemov")
 (set_attr "mode" "SF")])
 ;; A subset is vec_setv4sf.
 {
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])) << 4);
 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4SF")])
 (define_insn "*vec_setv4sf_sse4_1"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 {
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])) << 4);
 return "insertps\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sselog")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V4SF")])
 (define_insn "*avx_insertps"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(unspec:V4SF [(match_operand:V4SF 2 "nonimmediate_operand" "xm")
 		     UNSPEC_INSERTPS))]
 "TARGET_AVX"
 "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V4SF")])
 (define_insn "sse4_1_insertps"
 [(set (match_operand:V4SF 0 "register_operand" "=x")
 	(unspec:V4SF [(match_operand:V4SF 2 "register_operand" "x")
 		      (match_operand:SI 3 "const_0_to_255_operand" "n")]
 		     UNSPEC_INSERTPS))]
 "TARGET_SSE4_1"
 "insertps\t{%3, %2, %0|%0, %2, %3}";
 [(set_attr "type" "sselog")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V4SF")])
 (define_split
 [(set (match_operand:V4SF 0 "memory_operand" "")
 	(vec_merge:V4SF
 	  (match_operand:AVX256MODE4P 1 "register_operand" "x,x")
 	  (parallel [(const_int 0) (const_int 1)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x0, %1, %0|%0, %1, 0x0}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_hi_<mode>"
 	  (match_operand:AVX256MODE4P 1 "register_operand" "x,x")
 	  (parallel [(const_int 2) (const_int 3)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_lo_<mode>"
 	  (parallel [(const_int 0) (const_int 1)
 		     (const_int 2) (const_int 3)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_hi_<mode>"
 	  (parallel [(const_int 4) (const_int 5)
 		     (const_int 6) (const_int 7)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_lo_v16hi"
 		     (const_int 4) (const_int 5)
 		     (const_int 6) (const_int 7)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_hi_v16hi"
 		     (const_int 12) (const_int 13)
 		     (const_int 14) (const_int 15)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_lo_v32qi"
 		     (const_int 12) (const_int 13)
 		     (const_int 14) (const_int 15)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_extract_hi_v32qi"
 		     (const_int 28) (const_int 29)
 		     (const_int 30) (const_int 31)])))]
 "TARGET_AVX"
 "vextractf128\t{$0x1, %1, %0|%0, %1, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,store")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "*sse4_1_extractps"
 	  (match_operand:V4SF 1 "register_operand" "x")
 	  (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n")])))]
 "TARGET_SSE4_1"
 "%vextractps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "V4SF")])
 (define_insn_and_split "*vec_extract_v4sf_mem"
 [(set (match_operand:SF 0 "register_operand" "=x*rf")
 ;;
 ;; Parallel double-precision floating point element swizzling
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Recall that the 256-bit unpck insns only shuffle within their lanes.
 (define_insn "avx_unpckhpd256"
 [(set (match_operand:V4DF 0 "register_operand" "=x")
 	(vec_select:V4DF
 	  (vec_concat:V8DF
 	    (match_operand:V4DF 1 "register_operand" "x")
 "vunpckhpd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4DF")])
-(define_expand "sse2_unpckhpd_exp"
+(define_expand "vec_interleave_highv2df"
-[(set (match_operand:V2DF 0 "nonimmediate_operand" "")
+[(set (match_operand:V2DF 0 "register_operand" "")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
 	    (match_operand:V2DF 1 "nonimmediate_operand" "")
 	    (match_operand:V2DF 2 "nonimmediate_operand" ""))
 	  (parallel [(const_int 1)
 		     (const_int 3)])))]
 "TARGET_SSE2"
-"ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);")
+{
+if (!ix86_vec_interleave_v2df_operator_ok (operands, 1))
-(define_insn "*avx_unpckhpd"
+operands[2] = force_reg (V2DFmode, operands[2]);
-[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,m")
+})
+(define_insn "*avx_interleave_highv2df"
+[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,x,m")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" " x,o,x")
+	    (match_operand:V2DF 1 "nonimmediate_operand" " x,o,o,x")
-	    (match_operand:V2DF 2 "nonimmediate_operand" " x,x,0"))
+	    (match_operand:V2DF 2 "nonimmediate_operand" " x,1,x,0"))
 	  (parallel [(const_int 1)
 		     (const_int 3)])))]
-"TARGET_AVX && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+"TARGET_AVX && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
 "@
 vunpckhpd\t{%2, %1, %0|%0, %1, %2}
+vmovddup\t{%H1, %0|%0, %H1}
 vmovlpd\t{%H1, %2, %0|%0, %2, %H1}
 vmovhpd\t{%1, %0|%0, %1}"
-[(set_attr "type" "sselog,ssemov,ssemov")
+[(set_attr "type" "sselog,sselog,ssemov,ssemov")
 (set_attr "prefix" "vex")
-(set_attr "mode" "V2DF,V1DF,V1DF")])
+(set_attr "mode" "V2DF,V2DF,V1DF,V1DF")])
-(define_insn "sse2_unpckhpd"
+(define_insn "*sse3_interleave_highv2df"
+[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,x,m")
+	(vec_select:V2DF
+	  (vec_concat:V4DF
+	    (match_operand:V2DF 1 "nonimmediate_operand" " 0,o,o,x")
+	    (match_operand:V2DF 2 "nonimmediate_operand" " x,1,0,0"))
+	  (parallel [(const_int 1)
+		     (const_int 3)])))]
+"TARGET_SSE3 && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
+"@
+unpckhpd\t{%2, %0|%0, %2}
+movddup\t{%H1, %0|%0, %H1}
+movlpd\t{%H1, %0|%0, %H1}
+movhpd\t{%1, %0|%0, %1}"
+[(set_attr "type" "sselog,sselog,ssemov,ssemov")
+(set_attr "prefix_data16" "*,*,1,1")
+(set_attr "mode" "V2DF,V2DF,V1DF,V1DF")])
+(define_insn "*sse2_interleave_highv2df"
 [(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,m")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
 	    (match_operand:V2DF 1 "nonimmediate_operand" " 0,o,x")
 	    (match_operand:V2DF 2 "nonimmediate_operand" " x,0,0"))
 	  (parallel [(const_int 1)
 		     (const_int 3)])))]
-"TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+"TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
 "@
 unpckhpd\t{%2, %0|%0, %2}
 movlpd\t{%H1, %0|%0, %H1}
 movhpd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog,ssemov,ssemov")
+(set_attr "prefix_data16" "*,1,1")
 (set_attr "mode" "V2DF,V1DF,V1DF")])
-(define_insn "avx_movddup256"
+;; Recall that the 256-bit unpck insns only shuffle within their lanes.
-[(set (match_operand:V4DF 0 "register_operand" "=x")
+(define_expand "avx_movddup256"
+[(set (match_operand:V4DF 0 "register_operand" "")
 	(vec_select:V4DF
 	  (vec_concat:V8DF
-	    (match_operand:V4DF 1 "nonimmediate_operand" "xm")
+	    (match_operand:V4DF 1 "nonimmediate_operand" "")
 	    (match_dup 1))
-	  (parallel [(const_int 0) (const_int 2)
+	  (parallel [(const_int 0) (const_int 4)
-		     (const_int 4) (const_int 6)])))]
+		     (const_int 2) (const_int 6)])))]
 "TARGET_AVX"
-"vmovddup\t{%1, %0|%0, %1}"
+"")
-[(set_attr "type" "sselog1")
+(define_expand "avx_unpcklpd256"
+[(set (match_operand:V4DF 0 "register_operand" "")
+	(vec_select:V4DF
+	  (vec_concat:V8DF
+	    (match_operand:V4DF 1 "register_operand" "")
+	    (match_operand:V4DF 2 "nonimmediate_operand" ""))
+	  (parallel [(const_int 0) (const_int 4)
+		     (const_int 2) (const_int 6)])))]
+"TARGET_AVX"
+"")
+(define_insn "*avx_unpcklpd256"
+[(set (match_operand:V4DF 0 "register_operand"         "=x,x")
+	(vec_select:V4DF
+	  (vec_concat:V8DF
+	    (match_operand:V4DF 1 "nonimmediate_operand" "xm,x")
+	    (match_operand:V4DF 2 "nonimmediate_operand" " 1,xm"))
+	  (parallel [(const_int 0) (const_int 4)
+		     (const_int 2) (const_int 6)])))]
+"TARGET_AVX
+&& (!MEM_P (operands[1]) || rtx_equal_p (operands[1], operands[2]))"
+"@
+vmovddup\t{%1, %0|%0, %1}
+vunpcklpd\t{%2, %1, %0|%0, %1, %2}"
+[(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4DF")])
-(define_insn "*avx_movddup"
+(define_expand "vec_interleave_lowv2df"
-[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,o")
+[(set (match_operand:V2DF 0 "register_operand" "")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" "xm,x")
+	    (match_operand:V2DF 1 "nonimmediate_operand" "")
-	    (match_dup 1))
+	    (match_operand:V2DF 2 "nonimmediate_operand" ""))
 	  (parallel [(const_int 0)
 		     (const_int 2)])))]
-"TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+"TARGET_SSE2"
-"@
+{
-vmovddup\t{%1, %0|%0, %1}
+if (!ix86_vec_interleave_v2df_operator_ok (operands, 0))
-#"
+operands[1] = force_reg (V2DFmode, operands[1]);
-[(set_attr "type" "sselog1,ssemov")
+})
-(set_attr "prefix" "vex")
-(set_attr "mode" "V2DF")])
+(define_insn "*avx_interleave_lowv2df"
+[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,x,o")
-(define_insn "*sse3_movddup"
-[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,o")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" "xm,x")
+	    (match_operand:V2DF 1 "nonimmediate_operand" " x,m,x,0")
-	    (match_dup 1))
+	    (match_operand:V2DF 2 "nonimmediate_operand" " x,1,m,x"))
 	  (parallel [(const_int 0)
 		     (const_int 2)])))]
-"TARGET_SSE3 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+"TARGET_AVX && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
 "@
+vunpcklpd\t{%2, %1, %0|%0, %1, %2}
+vmovddup\t{%1, %0|%0, %1}
+vmovhpd\t{%2, %1, %0|%0, %1, %2}
+vmovlpd\t{%2, %H0|%H0, %2}"
+[(set_attr "type" "sselog,sselog,ssemov,ssemov")
+(set_attr "prefix" "vex")
+(set_attr "mode" "V2DF,V2DF,V1DF,V1DF")])
+(define_insn "*sse3_interleave_lowv2df"
+[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,x,o")
+	(vec_select:V2DF
+	  (vec_concat:V4DF
+	    (match_operand:V2DF 1 "nonimmediate_operand" " 0,m,0,0")
+	    (match_operand:V2DF 2 "nonimmediate_operand" " x,1,m,x"))
+	  (parallel [(const_int 0)
+		     (const_int 2)])))]
+"TARGET_SSE3 && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
+"@
+unpcklpd\t{%2, %0|%0, %2}
 movddup\t{%1, %0|%0, %1}
-#"
+movhpd\t{%2, %0|%0, %2}
-[(set_attr "type" "sselog1,ssemov")
+movlpd\t{%2, %H0|%H0, %2}"
-(set_attr "mode" "V2DF")])
+[(set_attr "type" "sselog,sselog,ssemov,ssemov")
+(set_attr "prefix_data16" "*,*,1,1")
+(set_attr "mode" "V2DF,V2DF,V1DF,V1DF")])
+(define_insn "*sse2_interleave_lowv2df"
+[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,o")
+	(vec_select:V2DF
+	  (vec_concat:V4DF
+	    (match_operand:V2DF 1 "nonimmediate_operand" " 0,0,0")
+	    (match_operand:V2DF 2 "nonimmediate_operand" " x,m,x"))
+	  (parallel [(const_int 0)
+		     (const_int 2)])))]
+"TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
+"@
+unpcklpd\t{%2, %0|%0, %2}
+movhpd\t{%2, %0|%0, %2}
+movlpd\t{%2, %H0|%H0, %2}"
+[(set_attr "type" "sselog,ssemov,ssemov")
+(set_attr "prefix_data16" "*,1,1")
+(set_attr "mode" "V2DF,V1DF,V1DF")])
 (define_split
 [(set (match_operand:V2DF 0 "memory_operand" "")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
 emit_move_insn (adjust_address (operands[0], DFmode, 0), low);
 emit_move_insn (adjust_address (operands[0], DFmode, 8), low);
 DONE;
 })
-(define_insn "avx_unpcklpd256"
+(define_split
-[(set (match_operand:V4DF 0 "register_operand" "=x")
+[(set (match_operand:V2DF 0 "register_operand" "")
-	(vec_select:V4DF
-	  (vec_concat:V8DF
-	    (match_operand:V4DF 1 "register_operand" "x")
-	    (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
-	  (parallel [(const_int 0) (const_int 4)
-		     (const_int 2) (const_int 6)])))]
-"TARGET_AVX"
-"vunpcklpd\t{%2, %1, %0|%0, %1, %2}"
-[(set_attr "type" "sselog")
-(set_attr "prefix" "vex")
-(set_attr "mode" "V4DF")])
-(define_expand "sse2_unpcklpd_exp"
-[(set (match_operand:V2DF 0 "nonimmediate_operand" "")
 	(vec_select:V2DF
 	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" "")
+	    (match_operand:V2DF 1 "memory_operand" "")
-	    (match_operand:V2DF 2 "nonimmediate_operand" ""))
+	    (match_dup 1))
-	  (parallel [(const_int 0)
+	  (parallel [(match_operand:SI 2 "const_0_to_1_operand" "")
-		     (const_int 2)])))]
+		     (match_operand:SI 3 "const_int_operand" "")])))]
-"TARGET_SSE2"
+"TARGET_SSE3 && INTVAL (operands[2]) + 2 == INTVAL (operands[3])"
-"ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);")
+[(set (match_dup 0) (vec_duplicate:V2DF (match_dup 1)))]
+{
-(define_insn "*avx_unpcklpd"
+operands[1] = adjust_address (operands[1], DFmode, INTVAL (operands[2]) * 8);
-[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,o")
+})
-	(vec_select:V2DF
-	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" " x,x,0")
-	    (match_operand:V2DF 2 "nonimmediate_operand" " x,m,x"))
-	  (parallel [(const_int 0)
-		     (const_int 2)])))]
-"TARGET_AVX && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-"@
-vunpcklpd\t{%2, %1, %0|%0, %1, %2}
-vmovhpd\t{%2, %1, %0|%0, %1, %2}
-vmovlpd\t{%2, %H0|%H0, %2}"
-[(set_attr "type" "sselog,ssemov,ssemov")
-(set_attr "prefix" "vex")
-(set_attr "mode" "V2DF,V1DF,V1DF")])
-(define_insn "sse2_unpcklpd"
-[(set (match_operand:V2DF 0 "nonimmediate_operand"     "=x,x,o")
-	(vec_select:V2DF
-	  (vec_concat:V4DF
-	    (match_operand:V2DF 1 "nonimmediate_operand" " 0,0,0")
-	    (match_operand:V2DF 2 "nonimmediate_operand" " x,m,x"))
-	  (parallel [(const_int 0)
-		     (const_int 2)])))]
-"TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-"@
-unpcklpd\t{%2, %0|%0, %2}
-movhpd\t{%2, %0|%0, %2}
-movlpd\t{%2, %H0|%H0, %2}"
-[(set_attr "type" "sselog,ssemov,ssemov")
-(set_attr "mode" "V2DF,V1DF,V1DF")])
 (define_expand "avx_shufpd256"
 [(match_operand:V4DF 0 "register_operand" "")
 (match_operand:V4DF 1 "register_operand" "")
 (match_operand:V4DF 2 "nonimmediate_operand" "")
 operands[3] = GEN_INT (mask);
 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V4DF")])
 (define_expand "sse2_shufpd"
 [(match_operand:V2DF 0 "register_operand" "")
 				GEN_INT (mask & 2 ? 3 : 2)));
 DONE;
 })
 (define_expand "vec_extract_even<mode>"
-[(set (match_operand:SSEMODE4S 0 "register_operand" "")
+[(match_operand:SSEMODE_EO 0 "register_operand" "")
-	(vec_select:SSEMODE4S
+(match_operand:SSEMODE_EO 1 "register_operand" "")
-	  (vec_concat:<ssedoublesizemode>
+(match_operand:SSEMODE_EO 2 "register_operand" "")]
-	    (match_operand:SSEMODE4S 1 "register_operand" "")
+""
-	    (match_operand:SSEMODE4S 2 "nonimmediate_operand" ""))
+{
-	  (parallel [(const_int 0)
+ix86_expand_vec_extract_even_odd (operands[0], operands[1], operands[2], 0);
-		     (const_int 2)
+DONE;
-		     (const_int 4)
+})
-		     (const_int 6)])))]
-"TARGET_SSE")
 (define_expand "vec_extract_odd<mode>"
-[(set (match_operand:SSEMODE4S 0 "register_operand" "")
+[(match_operand:SSEMODE_EO 0 "register_operand" "")
-	(vec_select:SSEMODE4S
+(match_operand:SSEMODE_EO 1 "register_operand" "")
-	  (vec_concat:<ssedoublesizemode>
+(match_operand:SSEMODE_EO 2 "register_operand" "")]
-	    (match_operand:SSEMODE4S 1 "register_operand" "")
+""
-	    (match_operand:SSEMODE4S 2 "nonimmediate_operand" ""))
+{
-	  (parallel [(const_int 1)
+ix86_expand_vec_extract_even_odd (operands[0], operands[1], operands[2], 1);
-		     (const_int 3)
+DONE;
-		     (const_int 5)
+})
-		     (const_int 7)])))]
-"TARGET_SSE")
-(define_expand "vec_extract_even<mode>"
-[(set (match_operand:SSEMODE2D 0 "register_operand" "")
-	(vec_select:SSEMODE2D
-	  (vec_concat:<ssedoublesizemode>
-	    (match_operand:SSEMODE2D 1 "register_operand" "")
-	    (match_operand:SSEMODE2D 2 "nonimmediate_operand" ""))
-	  (parallel [(const_int 0)
-	  	     (const_int 2)])))]
-"TARGET_SSE2")
-(define_expand "vec_extract_odd<mode>"
-[(set (match_operand:SSEMODE2D 0 "register_operand" "")
-	(vec_select:SSEMODE2D
-	  (vec_concat:<ssedoublesizemode>
-	    (match_operand:SSEMODE2D 1 "register_operand" "")
-	    (match_operand:SSEMODE2D 2 "nonimmediate_operand" ""))
-	  (parallel [(const_int 1)
-	  	     (const_int 3)])))]
-"TARGET_SSE2")
 ;; punpcklqdq and punpckhqdq are shorter than shufpd.
-(define_insn "*avx_punpckhqdq"
+(define_insn "*avx_interleave_highv2di"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(vec_select:V2DI
 	  (vec_concat:V4DI
 	    (match_operand:V2DI 1 "register_operand" "x")
 	    (match_operand:V2DI 2 "nonimmediate_operand" "xm"))
 "vpunpckhqdq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpckhqdq"
+(define_insn "vec_interleave_highv2di"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(vec_select:V2DI
 	  (vec_concat:V4DI
 	    (match_operand:V2DI 1 "register_operand" "0")
 	    (match_operand:V2DI 2 "nonimmediate_operand" "xm"))
 "punpckhqdq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpcklqdq"
+(define_insn "*avx_interleave_lowv2di"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(vec_select:V2DI
 	  (vec_concat:V4DI
 	    (match_operand:V2DI 1 "register_operand" "x")
 	    (match_operand:V2DI 2 "nonimmediate_operand" "xm"))
 "vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpcklqdq"
+(define_insn "vec_interleave_lowv2di"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(vec_select:V2DI
 	  (vec_concat:V4DI
 	    (match_operand:V2DI 1 "register_operand" "0")
 	    (match_operand:V2DI 2 "nonimmediate_operand" "xm"))
 operands[3] = GEN_INT (mask);
 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V2DF")])
 (define_insn "sse2_shufpd_<mode>"
 [(set (match_operand:SSEMODE2D 0 "register_operand" "=x")
 operands[3] = GEN_INT (mask);
 return "shufpd\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "V2DF")])
 ;; Avoid combining registers from different units in a single alternative,
 ;; see comment above inline_secondary_memory_needed function in i386.c
 (define_insn "*avx_storehpd"
 unpckhpd\t%0, %0
 #
 #
 #"
 [(set_attr "type" "ssemov,sselog1,ssemov,fmov,imov")
+(set_attr "prefix_data16" "1,*,*,*,*")
 (set_attr "mode" "V1DF,V2DF,DF,DF,DF")])
 (define_split
 [(set (match_operand:DF 0 "register_operand" "")
 	(vec_select:DF
 #
 #
 #
 #"
 [(set_attr "type" "ssemov,ssemov,ssemov,fmov,imov")
+(set_attr "prefix_data16" "1,*,*,*,*")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "V1DF,DF,DF,DF,DF")])
 (define_split
 [(set (match_operand:DF 0 "register_operand" "")
 shufpd\t{$1, %1, %0|%0, %1, 1}
 #
 #
 #"
 [(set_attr "type" "ssemov,sselog,sselog,ssemov,fmov,imov")
+(set_attr "prefix_data16" "1,*,*,*,*,*")
+(set_attr "length_immediate" "*,*,1,*,*,*")
 (set_attr "mode" "V1DF,V2DF,V2DF,DF,DF,DF")])
 (define_split
 [(set (match_operand:V2DF 0 "memory_operand" "")
 	(vec_concat:V2DF
 movhpd\t{%H1, %0|%0, %H1}
 #
 #
 #"
 [(set_attr "type" "ssemov,ssemov,ssemov,sselog,ssemov,ssemov,fmov,imov")
+(set_attr "prefix_data16" "*,1,*,*,1,*,*,*")
+(set_attr "length_immediate" "*,*,*,1,*,*,*,*")
 (set_attr "mode" "DF,V1DF,V1DF,V2DF,V1DF,DF,DF,DF")])
 (define_split
 [(set (match_operand:V2DF 0 "memory_operand" "")
 	(vec_concat:V2DF
 movlpd\t{%2, %0|%0, %2}
 shufpd\t{$2, %2, %0|%0, %2, 2}
 movhps\t{%H1, %0|%0, %H1}
 movhps\t{%1, %H0|%H0, %1}"
 [(set_attr "type" "ssemov,ssemov,ssemov,sselog,ssemov,ssemov")
+(set_attr "prefix_data16" "*,1,1,*,*,*")
+(set_attr "length_immediate" "*,*,*,1,*,*")
 (set_attr "mode" "DF,V1DF,V1DF,V2DF,V1DF,V1DF")])
 (define_insn "*vec_dupv2df_sse3"
 [(set (match_operand:V2DF 0 "register_operand" "=x")
 	(vec_duplicate:V2DF
 movhpd\t{%2, %0|%0, %2}
 movsd\t{%1, %0|%0, %1}
 movlhps\t{%2, %0|%0, %2}
 movhps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog,ssemov,ssemov,ssemov,ssemov")
+(set_attr "prefix_data16" "*,1,*,*,*")
 (set_attr "mode" "V2DF,V1DF,DF,V4SF,V2SF")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Parallel integral arithmetic
 (define_insn_and_split "mulv16qi3"
 [(set (match_operand:V16QI 0 "register_operand" "")
 	(mult:V16QI (match_operand:V16QI 1 "register_operand" "")
 		    (match_operand:V16QI 2 "register_operand" "")))]
 "TARGET_SSE2
-&& !(reload_completed || reload_in_progress)"
+&& can_create_pseudo_p ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
-rtx t[12], op0, op[3];
+rtx t[6];
 int i;
-if (TARGET_SSE5)
+for (i = 0; i < 6; ++i)
-{
-/* On SSE5, we can take advantage of the pperm instruction to pack and
-	 unpack the bytes.  Unpack data such that we've got a source byte in
-	 each low byte of each word.  We don't care what goes into the high
-	 byte, so put 0 there.  */
-for (i = 0; i < 6; ++i)
-t[i] = gen_reg_rtx (V8HImode);
-for (i = 0; i < 2; i++)
-{
-op[0] = t[i];
-op[1] = operands[i+1];
-ix86_expand_sse5_unpack (op, true, true);		/* high bytes */
-op[0] = t[i+2];
-ix86_expand_sse5_unpack (op, true, false);		/* low bytes */
-}
-/* Multiply words.  */
-emit_insn (gen_mulv8hi3 (t[4], t[0], t[1]));		/* high bytes */
-emit_insn (gen_mulv8hi3 (t[5], t[2], t[3]));		/* low  bytes */
-/* Pack the low byte of each word back into a single xmm */
-op[0] = operands[0];
-op[1] = t[5];
-op[2] = t[4];
-ix86_expand_sse5_pack (op);
-DONE;
-}
-for (i = 0; i < 12; ++i)
 t[i] = gen_reg_rtx (V16QImode);
 /* Unpack data such that we've got a source byte in each low byte of
 each word.  We don't care what goes into the high byte of each word.
 Rather than trying to get zero in there, most convenient is to let
 it be a copy of the low byte.  */
-emit_insn (gen_sse2_punpckhbw (t[0], operands[1], operands[1]));
+emit_insn (gen_vec_interleave_highv16qi (t[0], operands[1], operands[1]));
-emit_insn (gen_sse2_punpckhbw (t[1], operands[2], operands[2]));
+emit_insn (gen_vec_interleave_highv16qi (t[1], operands[2], operands[2]));
-emit_insn (gen_sse2_punpcklbw (t[2], operands[1], operands[1]));
+emit_insn (gen_vec_interleave_lowv16qi (t[2], operands[1], operands[1]));
-emit_insn (gen_sse2_punpcklbw (t[3], operands[2], operands[2]));
+emit_insn (gen_vec_interleave_lowv16qi (t[3], operands[2], operands[2]));
 /* Multiply words.  The end-of-line annotations here give a picture of what
 the output of that instruction looks like.  Dot means don't care; the
 letters are the bytes of the result with A being the most significant.  */
 emit_insn (gen_mulv8hi3 (gen_lowpart (V8HImode, t[4]), /* .A.B.C.D.E.F.G.H */
 			   gen_lowpart (V8HImode, t[1])));
 emit_insn (gen_mulv8hi3 (gen_lowpart (V8HImode, t[5]), /* .I.J.K.L.M.N.O.P */
 			   gen_lowpart (V8HImode, t[2]),
 			   gen_lowpart (V8HImode, t[3])));
-/* Extract the relevant bytes and merge them back together.  */
+/* Extract the even bytes and merge them back together.  */
-emit_insn (gen_sse2_punpckhbw (t[6], t[5], t[4]));	/* ..AI..BJ..CK..DL */
+ix86_expand_vec_extract_even_odd (operands[0], t[5], t[4], 0);
-emit_insn (gen_sse2_punpcklbw (t[7], t[5], t[4]));	/* ..EM..FN..GO..HP */
-emit_insn (gen_sse2_punpckhbw (t[8], t[7], t[6]));	/* ....AEIM....BFJN */
-emit_insn (gen_sse2_punpcklbw (t[9], t[7], t[6]));	/* ....CGKO....DHLP */
-emit_insn (gen_sse2_punpckhbw (t[10], t[9], t[8]));	/* ........ACEGIKMO */
-emit_insn (gen_sse2_punpcklbw (t[11], t[9], t[8]));	/* ........BDFHJLNP */
-op0 = operands[0];
-emit_insn (gen_sse2_punpcklbw (op0, t[11], t[10]));	/* ABCDEFGHIJKLMNOP */
 DONE;
 })
 (define_expand "mulv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "")
 	      (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	      (parallel [(const_int 0) (const_int 2)])))))]
 "TARGET_AVX && ix86_binary_operator_ok (MULT, V4SImode, operands)"
 "vpmuldq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseimul")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_mulv2siv2di3"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 			   (const_int 5)
 			   (const_int 7)]))))))]
 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, V8HImode, operands)"
 "pmaddwd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "simul")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_expand "mulv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "")
 	(mult:V4SI (match_operand:V4SI 1 "register_operand" "")
 		   (match_operand:V4SI 2 "register_operand" "")))]
 "TARGET_SSE2"
 {
-if (TARGET_SSE4_1 || TARGET_SSE5)
+if (TARGET_SSE4_1 || TARGET_AVX)
 ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);
 })
 (define_insn "*avx_mulv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(mult:V4SI (match_operand:V4SI 1 "nonimmediate_operand" "%x")
 		   (match_operand:V4SI 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX && ix86_binary_operator_ok (MULT, V4SImode, operands)"
 "vpmulld\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseimul")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_mulv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 "pmulld\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseimul")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
-;; We don't have a straight 32-bit parallel multiply on SSE5, so fake it with a
-;; multiply/add.  In general, we expect the define_split to occur before
-;; register allocation, so we have to handle the corner case where the target
-;; is the same as one of the inputs.
-(define_insn_and_split "*sse5_mulv4si3"
-[(set (match_operand:V4SI 0 "register_operand" "=&x")
-	(mult:V4SI (match_operand:V4SI 1 "register_operand" "%x")
-		   (match_operand:V4SI 2 "nonimmediate_operand" "xm")))]
-"TARGET_SSE5"
-"#"
-"&& (reload_completed
-|| (!reg_mentioned_p (operands[0], operands[1])
-	   && !reg_mentioned_p (operands[0], operands[2])))"
-[(set (match_dup 0)
-	(match_dup 3))
-(set (match_dup 0)
-	(plus:V4SI (mult:V4SI (match_dup 1)
-			      (match_dup 2))
-		   (match_dup 0)))]
-{
-operands[3] = CONST0_RTX (V4SImode);
-}
-[(set_attr "type" "ssemuladd")
-(set_attr "mode" "TI")])
 (define_insn_and_split "*sse2_mulv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "")
 	(mult:V4SI (match_operand:V4SI 1 "register_operand" "")
 		   (match_operand:V4SI 2 "register_operand" "")))]
-"TARGET_SSE2 && !TARGET_SSE4_1 && !TARGET_SSE5
+"TARGET_SSE2 && !TARGET_SSE4_1 && !TARGET_AVX
-&& !(reload_completed || reload_in_progress)"
+&& can_create_pseudo_p ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 rtx t1, t2, t3, t4, t5, t6, thirtytwo;
 				     op1, op2));
 /* Shift both input vectors down one element, so that elements 3
 and 1 are now in the slots for elements 2 and 0.  For K8, at
 least, this is faster than using a shuffle.  */
-emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, t2),
+emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, t2),
-			       gen_lowpart (TImode, op1),
+				 gen_lowpart (V1TImode, op1),
-			       thirtytwo));
+				 thirtytwo));
-emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, t3),
+emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, t3),
-			       gen_lowpart (TImode, op2),
+				 gen_lowpart (V1TImode, op2),
-			       thirtytwo));
+				 thirtytwo));
 /* Multiply elements 3 and 1.  */
 emit_insn (gen_sse2_umulv2siv2di3 (gen_lowpart (V2DImode, t4),
 				     t2, t3));
 /* Move the results in element 2 down to element 1; we don't care
 				const0_rtx, const0_rtx));
 emit_insn (gen_sse2_pshufd_1 (t6, t4, const0_rtx, const2_rtx,
 				const0_rtx, const0_rtx));
 /* Merge the parts back together.  */
-emit_insn (gen_sse2_punpckldq (op0, t5, t6));
+emit_insn (gen_vec_interleave_lowv4si (op0, t5, t6));
 DONE;
 })
 (define_insn_and_split "mulv2di3"
 [(set (match_operand:V2DI 0 "register_operand" "")
 	(mult:V2DI (match_operand:V2DI 1 "register_operand" "")
 		   (match_operand:V2DI 2 "register_operand" "")))]
 "TARGET_SSE2
-&& !(reload_completed || reload_in_progress)"
+&& can_create_pseudo_p ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 rtx t1, t2, t3, t4, t5, t6, thirtytwo;
 rtx op0, op1, op2;
-if (TARGET_SSE5)
+op0 = operands[0];
+op1 = operands[1];
+op2 = operands[2];
+if (TARGET_XOP)
 {
 /* op1: A,B,C,D, op2: E,F,G,H */
-op0 = operands[0];
+op1 = gen_lowpart (V4SImode, op1);
-op1 = gen_lowpart (V4SImode, operands[1]);
+op2 = gen_lowpart (V4SImode, op2);
-op2 = gen_lowpart (V4SImode, operands[2]);
 t1 = gen_reg_rtx (V4SImode);
 t2 = gen_reg_rtx (V4SImode);
-t3 = gen_reg_rtx (V4SImode);
+t3 = gen_reg_rtx (V2DImode);
 t4 = gen_reg_rtx (V2DImode);
-t5 = gen_reg_rtx (V2DImode);
 /* t1: B,A,D,C */
 emit_insn (gen_sse2_pshufd_1 (t1, op1,
 				    GEN_INT (1),
 				    GEN_INT (0),
 				    GEN_INT (3),
 				    GEN_INT (2)));
-/* t2: 0 */
+/* t2: (B*E),(A*F),(D*G),(C*H) */
-emit_move_insn (t2, CONST0_RTX (V4SImode));
+emit_insn (gen_mulv4si3 (t2, t1, op2));
-/* t3: (B*E),(A*F),(D*G),(C*H) */
-emit_insn (gen_sse5_pmacsdd (t3, t1, op2, t2));
 /* t4: (B*E)+(A*F), (D*G)+(C*H) */
-emit_insn (gen_sse5_phadddq (t4, t3));
+emit_insn (gen_xop_phadddq (t3, t2));
 /* t5: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
-emit_insn (gen_ashlv2di3 (t5, t4, GEN_INT (32)));
+emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
-emit_insn (gen_sse5_pmacsdql (op0, op1, op2, t5));
+emit_insn (gen_xop_pmacsdql (op0, op1, op2, t4));
-DONE;
 }
+else
-op0 = operands[0];
+{
-op1 = operands[1];
+t1 = gen_reg_rtx (V2DImode);
-op2 = operands[2];
+t2 = gen_reg_rtx (V2DImode);
-t1 = gen_reg_rtx (V2DImode);
+t3 = gen_reg_rtx (V2DImode);
-t2 = gen_reg_rtx (V2DImode);
+t4 = gen_reg_rtx (V2DImode);
-t3 = gen_reg_rtx (V2DImode);
+t5 = gen_reg_rtx (V2DImode);
-t4 = gen_reg_rtx (V2DImode);
+t6 = gen_reg_rtx (V2DImode);
-t5 = gen_reg_rtx (V2DImode);
+thirtytwo = GEN_INT (32);
-t6 = gen_reg_rtx (V2DImode);
-thirtytwo = GEN_INT (32);
+/* Multiply low parts.  */
+emit_insn (gen_sse2_umulv2siv2di3 (t1, gen_lowpart (V4SImode, op1),
-/* Multiply low parts.  */
+				         gen_lowpart (V4SImode, op2)));
-emit_insn (gen_sse2_umulv2siv2di3 (t1, gen_lowpart (V4SImode, op1),
-				     gen_lowpart (V4SImode, op2)));
+/* Shift input vectors left 32 bits so we can multiply high parts.  */
+emit_insn (gen_lshrv2di3 (t2, op1, thirtytwo));
-/* Shift input vectors left 32 bits so we can multiply high parts.  */
+emit_insn (gen_lshrv2di3 (t3, op2, thirtytwo));
-emit_insn (gen_lshrv2di3 (t2, op1, thirtytwo));
-emit_insn (gen_lshrv2di3 (t3, op2, thirtytwo));
+/* Multiply high parts by low parts.  */
+emit_insn (gen_sse2_umulv2siv2di3 (t4, gen_lowpart (V4SImode, op1),
-/* Multiply high parts by low parts.  */
+					 gen_lowpart (V4SImode, t3)));
-emit_insn (gen_sse2_umulv2siv2di3 (t4, gen_lowpart (V4SImode, op1),
+emit_insn (gen_sse2_umulv2siv2di3 (t5, gen_lowpart (V4SImode, op2),
-				     gen_lowpart (V4SImode, t3)));
+					 gen_lowpart (V4SImode, t2)));
-emit_insn (gen_sse2_umulv2siv2di3 (t5, gen_lowpart (V4SImode, op2),
-				     gen_lowpart (V4SImode, t2)));
+/* Shift them back.  */
+emit_insn (gen_ashlv2di3 (t4, t4, thirtytwo));
-/* Shift them back.  */
+emit_insn (gen_ashlv2di3 (t5, t5, thirtytwo));
-emit_insn (gen_ashlv2di3 (t4, t4, thirtytwo));
-emit_insn (gen_ashlv2di3 (t5, t5, thirtytwo));
+/* Add the three parts together.  */
+emit_insn (gen_addv2di3 (t6, t1, t4));
-/* Add the three parts together.  */
+emit_insn (gen_addv2di3 (op0, t6, t5));
-emit_insn (gen_addv2di3 (t6, t1, t4));
+}
-emit_insn (gen_addv2di3 (op0, t6, t5));
 DONE;
 })
 (define_expand "vec_widen_smult_hi_v8hi"
 [(match_operand:V4SI 0 "register_operand" "")
 (define_expand "vec_widen_smult_hi_v4si"
 [(match_operand:V2DI 0 "register_operand" "")
 (match_operand:V4SI 1 "register_operand" "")
 (match_operand:V4SI 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtx t1, t2;
 t1 = gen_reg_rtx (V4SImode);
 t2 = gen_reg_rtx (V4SImode);
 emit_insn (gen_sse2_pshufd_1 (t2, operands[2],
 				GEN_INT (0),
 				GEN_INT (2),
 				GEN_INT (1),
 				GEN_INT (3)));
-emit_insn (gen_sse5_mulv2div2di3_high (operands[0], t1, t2));
+emit_insn (gen_xop_mulv2div2di3_high (operands[0], t1, t2));
 DONE;
 })
 (define_expand "vec_widen_smult_lo_v4si"
 [(match_operand:V2DI 0 "register_operand" "")
 (match_operand:V4SI 1 "register_operand" "")
 (match_operand:V4SI 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtx t1, t2;
 t1 = gen_reg_rtx (V4SImode);
 t2 = gen_reg_rtx (V4SImode);
 emit_insn (gen_sse2_pshufd_1 (t2, operands[2],
 				GEN_INT (0),
 				GEN_INT (2),
 				GEN_INT (1),
 				GEN_INT (3)));
-emit_insn (gen_sse5_mulv2div2di3_low (operands[0], t1, t2));
+emit_insn (gen_xop_mulv2div2di3_low (operands[0], t1, t2));
-DONE;
 DONE;
 })
 (define_expand "vec_widen_umult_hi_v4si"
 [(match_operand:V2DI 0 "register_operand" "")
 emit_insn (gen_sse2_umulv2siv2di3 (t1, operands[1], operands[2]));
 emit_insn (gen_addv2di3 (t1, t1, operands[3]));
 t2 = gen_reg_rtx (V4SImode);
 t3 = gen_reg_rtx (V4SImode);
-emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, t2),
+emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, t2),
-gen_lowpart (TImode, operands[1]),
+				 gen_lowpart (V1TImode, operands[1]),
-GEN_INT (32)));
+				 GEN_INT (32)));
-emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, t3),
+emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, t3),
-gen_lowpart (TImode, operands[2]),
+				 gen_lowpart (V1TImode, operands[2]),
-GEN_INT (32)));
+				 GEN_INT (32)));
 t4 = gen_reg_rtx (V2DImode);
 emit_insn (gen_sse2_umulv2siv2di3 (t4, t2, t3));
 emit_insn (gen_addv2di3 (operands[0], t1, t4));
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_AVX"
 "vpsra<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix" "vex")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
 (set_attr "mode" "TI")])
 (define_insn "ashr<mode>3"
 [(set (match_operand:SSEMODE24 0 "register_operand" "=x")
 	(ashiftrt:SSEMODE24
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_SSE2"
 "psra<ssevecsize>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix_data16" "1")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
+(set_attr "mode" "TI")])
+(define_insn "*avx_lshrv1ti3"
+[(set (match_operand:V1TI 0 "register_operand" "=x")
+	(lshiftrt:V1TI
+	 (match_operand:V1TI 1 "register_operand" "x")
+	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
+"TARGET_AVX"
+{
+operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
+return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
+}
+[(set_attr "type" "sseishft")
+(set_attr "prefix" "vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*avx_lshr<mode>3"
 [(set (match_operand:SSEMODE248 0 "register_operand" "=x")
 	(lshiftrt:SSEMODE248
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_AVX"
 "vpsrl<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix" "vex")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
+(set_attr "mode" "TI")])
+(define_insn "sse2_lshrv1ti3"
+[(set (match_operand:V1TI 0 "register_operand" "=x")
+	(lshiftrt:V1TI
+	 (match_operand:V1TI 1 "register_operand" "0")
+	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
+"TARGET_SSE2"
+{
+operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
+return "psrldq\t{%2, %0|%0, %2}";
+}
+[(set_attr "type" "sseishft")
+(set_attr "prefix_data16" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "lshr<mode>3"
 [(set (match_operand:SSEMODE248 0 "register_operand" "=x")
 	(lshiftrt:SSEMODE248
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_SSE2"
 "psrl<ssevecsize>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix_data16" "1")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
+(set_attr "mode" "TI")])
+(define_insn "*avx_ashlv1ti3"
+[(set (match_operand:V1TI 0 "register_operand" "=x")
+	(ashift:V1TI (match_operand:V1TI 1 "register_operand" "x")
+		     (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
+"TARGET_AVX"
+{
+operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
+return "vpslldq\t{%2, %1, %0|%0, %1, %2}";
+}
+[(set_attr "type" "sseishft")
+(set_attr "prefix" "vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*avx_ashl<mode>3"
 [(set (match_operand:SSEMODE248 0 "register_operand" "=x")
 	(ashift:SSEMODE248
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_AVX"
 "vpsll<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix" "vex")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
+(set_attr "mode" "TI")])
+(define_insn "sse2_ashlv1ti3"
+[(set (match_operand:V1TI 0 "register_operand" "=x")
+	(ashift:V1TI (match_operand:V1TI 1 "register_operand" "0")
+		     (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
+"TARGET_SSE2"
+{
+operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
+return "pslldq\t{%2, %0|%0, %2}";
+}
+[(set_attr "type" "sseishft")
+(set_attr "prefix_data16" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "ashl<mode>3"
 [(set (match_operand:SSEMODE248 0 "register_operand" "=x")
 	(ashift:SSEMODE248
 	  (match_operand:SI 2 "nonmemory_operand" "xN")))]
 "TARGET_SSE2"
 "psll<ssevecsize>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseishft")
 (set_attr "prefix_data16" "1")
+(set (attr "length_immediate")
+(if_then_else (match_operand 2 "const_int_operand" "")
+(const_string "1")
+(const_string "0")))
 (set_attr "mode" "TI")])
 (define_expand "vec_shl_<mode>"
 [(set (match_operand:SSEMODEI 0 "register_operand" "")
-(ashift:TI (match_operand:SSEMODEI 1 "register_operand" "")
+(ashift:V1TI
-		   (match_operand:SI 2 "const_0_to_255_mul_8_operand" "")))]
+	 (match_operand:SSEMODEI 1 "register_operand" "")
-"TARGET_SSE2"
+	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "")))]
-{
+"TARGET_SSE2"
-operands[0] = gen_lowpart (TImode, operands[0]);
+{
-operands[1] = gen_lowpart (TImode, operands[1]);
+operands[0] = gen_lowpart (V1TImode, operands[0]);
+operands[1] = gen_lowpart (V1TImode, operands[1]);
 })
 (define_expand "vec_shr_<mode>"
 [(set (match_operand:SSEMODEI 0 "register_operand" "")
-(lshiftrt:TI (match_operand:SSEMODEI 1 "register_operand" "")
+(lshiftrt:V1TI
-		     (match_operand:SI 2 "const_0_to_255_mul_8_operand" "")))]
+	 (match_operand:SSEMODEI 1 "register_operand" "")
-"TARGET_SSE2"
+	 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "")))]
-{
+"TARGET_SSE2"
-operands[0] = gen_lowpart (TImode, operands[0]);
+{
-operands[1] = gen_lowpart (TImode, operands[1]);
+operands[0] = gen_lowpart (V1TImode, operands[0]);
+operands[1] = gen_lowpart (V1TImode, operands[1]);
 })
 (define_insn "*avx_<code><mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(maxmin:SSEMODE124
 	  (match_operand:SSEMODE124 1 "nonimmediate_operand" "%x")
 	  (match_operand:SSEMODE124 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 "vp<maxminiprefix><ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set (attr "prefix_extra")
+(if_then_else
+(ne (symbol_ref "<MODE>mode != ((<CODE> == SMAX || <CODE> == SMIN) ? V8HImode : V16QImode)")
+	   (const_int 0))
+(const_string "1")
+(const_string "0")))
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_expand "<code>v16qi3"
 [(set (match_operand:V16QI 0 "register_operand" "")
 (define_expand "sse2_eq<mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "")
 	(eq:SSEMODE124
 	  (match_operand:SSEMODE124 1 "nonimmediate_operand" "")
 	  (match_operand:SSEMODE124 2 "nonimmediate_operand" "")))]
-"TARGET_SSE2 && !TARGET_SSE5"
+"TARGET_SSE2 && !TARGET_XOP "
 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
 (define_insn "*avx_eq<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(eq:SSEMODE1248
 	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "%x")
 	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
 "vpcmpeq<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "ssecmp")
+(set (attr "prefix_extra")
+(if_then_else (match_operand:V2DI 0 "" "")
+(const_string "1")
+(const_string "*")))
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse2_eq<mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(eq:SSEMODE124
 	  (match_operand:SSEMODE124 1 "nonimmediate_operand" "%0")
 	  (match_operand:SSEMODE124 2 "nonimmediate_operand" "xm")))]
-"TARGET_SSE2 && !TARGET_SSE5
+"TARGET_SSE2 && !TARGET_XOP
 && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
 "pcmpeq<ssevecsize>\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecmp")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 	  (match_operand:SSEMODE1248 1 "register_operand" "x")
 	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX"
 "vpcmpgt<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "ssecmp")
+(set (attr "prefix_extra")
+(if_then_else (match_operand:V2DI 0 "" "")
+(const_string "1")
+(const_string "*")))
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "sse2_gt<mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(gt:SSEMODE124
 	  (match_operand:SSEMODE124 1 "register_operand" "0")
 	  (match_operand:SSEMODE124 2 "nonimmediate_operand" "xm")))]
-"TARGET_SSE2 && !TARGET_SSE5"
+"TARGET_SSE2 && !TARGET_XOP"
 "pcmpgt<ssevecsize>\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecmp")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 	  (match_operand:V2DI 1 "register_operand" "0")
 	  (match_operand:V2DI 2 "nonimmediate_operand" "xm")))]
 "TARGET_SSE4_2"
 "pcmpgtq\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecmp")
+(set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_expand "vcond<mode>"
 [(set (match_operand:SSEMODE124C8 0 "register_operand" "")
 (if_then_else:SSEMODE124C8
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_expand "<code><mode>3"
 [(set (match_operand:SSEMODEI 0 "register_operand" "")
-	(plogic:SSEMODEI
+	(any_logic:SSEMODEI
 	  (match_operand:SSEMODEI 1 "nonimmediate_operand" "")
 	  (match_operand:SSEMODEI 2 "nonimmediate_operand" "")))]
 "TARGET_SSE"
 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
 (define_insn "*avx_<code><mode>3"
 [(set (match_operand:AVX256MODEI 0 "register_operand" "=x")
-(plogic:AVX256MODEI
+(any_logic:AVX256MODEI
 (match_operand:AVX256MODEI 1 "nonimmediate_operand" "%x")
 (match_operand:AVX256MODEI 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"v<plogicprefix>ps\t{%2, %1, %0|%0, %1, %2}"
+"v<logicprefix>ps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecpsmode>")])
 (define_insn "*sse_<code><mode>3"
 [(set (match_operand:SSEMODEI 0 "register_operand" "=x")
-(plogic:SSEMODEI
+(any_logic:SSEMODEI
 (match_operand:SSEMODEI 1 "nonimmediate_operand" "%0")
 (match_operand:SSEMODEI 2 "nonimmediate_operand" "xm")))]
 "(TARGET_SSE && !TARGET_SSE2)
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"<plogicprefix>ps\t{%2, %0|%0, %2}"
+"<logicprefix>ps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "mode" "V4SF")])
 (define_insn "*avx_<code><mode>3"
 [(set (match_operand:SSEMODEI 0 "register_operand" "=x")
-(plogic:SSEMODEI
+(any_logic:SSEMODEI
 (match_operand:SSEMODEI 1 "nonimmediate_operand" "%x")
 (match_operand:SSEMODEI 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"vp<plogicprefix>\t{%2, %1, %0|%0, %1, %2}"
+"vp<logicprefix>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse2_<code><mode>3"
 [(set (match_operand:SSEMODEI 0 "register_operand" "=x")
-	(plogic:SSEMODEI
+	(any_logic:SSEMODEI
 	  (match_operand:SSEMODEI 1 "nonimmediate_operand" "%0")
 	  (match_operand:SSEMODEI 2 "nonimmediate_operand" "xm")))]
 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
-"p<plogicprefix>\t{%2, %0|%0, %2}"
+"p<logicprefix>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_expand "<code>tf3"
 [(set (match_operand:TF 0 "register_operand" "")
-	(plogic:TF
+	(any_logic:TF
 	  (match_operand:TF 1 "nonimmediate_operand" "")
 	  (match_operand:TF 2 "nonimmediate_operand" "")))]
 "TARGET_SSE2"
 "ix86_fixup_binary_operands_no_copy (<CODE>, TFmode, operands);")
 (define_insn "*<code>tf3"
 [(set (match_operand:TF 0 "register_operand" "=x")
-	(plogic:TF
+	(any_logic:TF
 	  (match_operand:TF 1 "nonimmediate_operand" "%0")
 	  (match_operand:TF 2 "nonimmediate_operand" "xm")))]
 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, TFmode, operands)"
-"p<plogicprefix>\t{%2, %0|%0, %2}"
+"p<logicprefix>\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Parallel integral element swizzling
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Reduce:
-;;      op1 = abcdefghijklmnop
-;;      op2 = qrstuvwxyz012345
-;;       h1 = aqbrcsdteufvgwhx
-;;       l1 = iyjzk0l1m2n3o4p5
-;;       h2 = aiqybjrzcks0dlt1
-;;       l2 = emu2fnv3gow4hpx5
-;;       h3 = aeimquy2bfjnrvz3
-;;       l3 = cgkosw04dhlptx15
-;;   result = bdfhjlnprtvxz135
 (define_expand "vec_pack_trunc_v8hi"
 [(match_operand:V16QI 0 "register_operand" "")
 (match_operand:V8HI 1 "register_operand" "")
 (match_operand:V8HI 2 "register_operand" "")]
 "TARGET_SSE2"
 {
-rtx op1, op2, h1, l1, h2, l2, h3, l3;
+rtx op1 = gen_lowpart (V16QImode, operands[1]);
+rtx op2 = gen_lowpart (V16QImode, operands[2]);
-if (TARGET_SSE5)
+ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
-{
-ix86_expand_sse5_pack (operands);
-DONE;
-}
-op1 = gen_lowpart (V16QImode, operands[1]);
-op2 = gen_lowpart (V16QImode, operands[2]);
-h1 = gen_reg_rtx (V16QImode);
-l1 = gen_reg_rtx (V16QImode);
-h2 = gen_reg_rtx (V16QImode);
-l2 = gen_reg_rtx (V16QImode);
-h3 = gen_reg_rtx (V16QImode);
-l3 = gen_reg_rtx (V16QImode);
-emit_insn (gen_vec_interleave_highv16qi (h1, op1, op2));
-emit_insn (gen_vec_interleave_lowv16qi (l1, op1, op2));
-emit_insn (gen_vec_interleave_highv16qi (h2, l1, h1));
-emit_insn (gen_vec_interleave_lowv16qi (l2, l1, h1));
-emit_insn (gen_vec_interleave_highv16qi (h3, l2, h2));
-emit_insn (gen_vec_interleave_lowv16qi (l3, l2, h2));
-emit_insn (gen_vec_interleave_lowv16qi (operands[0], l3, h3));
 DONE;
 })
-;; Reduce:
-;;      op1 = abcdefgh
-;;      op2 = ijklmnop
-;;       h1 = aibjckdl
-;;       l1 = emfngohp
-;;       h2 = aeimbfjn
-;;       l2 = cgkodhlp
-;;   result = bdfhjlnp
 (define_expand "vec_pack_trunc_v4si"
 [(match_operand:V8HI 0 "register_operand" "")
 (match_operand:V4SI 1 "register_operand" "")
 (match_operand:V4SI 2 "register_operand" "")]
 "TARGET_SSE2"
 {
-rtx op1, op2, h1, l1, h2, l2;
+rtx op1 = gen_lowpart (V8HImode, operands[1]);
+rtx op2 = gen_lowpart (V8HImode, operands[2]);
-if (TARGET_SSE5)
+ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
-{
-ix86_expand_sse5_pack (operands);
-DONE;
-}
-op1 = gen_lowpart (V8HImode, operands[1]);
-op2 = gen_lowpart (V8HImode, operands[2]);
-h1 = gen_reg_rtx (V8HImode);
-l1 = gen_reg_rtx (V8HImode);
-h2 = gen_reg_rtx (V8HImode);
-l2 = gen_reg_rtx (V8HImode);
-emit_insn (gen_vec_interleave_highv8hi (h1, op1, op2));
-emit_insn (gen_vec_interleave_lowv8hi (l1, op1, op2));
-emit_insn (gen_vec_interleave_highv8hi (h2, l1, h1));
-emit_insn (gen_vec_interleave_lowv8hi (l2, l1, h1));
-emit_insn (gen_vec_interleave_lowv8hi (operands[0], l2, h2));
 DONE;
 })
-;; Reduce:
-;;     op1 = abcd
-;;     op2 = efgh
-;;      h1 = aebf
-;;      l1 = cgdh
-;;  result = bdfh
 (define_expand "vec_pack_trunc_v2di"
 [(match_operand:V4SI 0 "register_operand" "")
 (match_operand:V2DI 1 "register_operand" "")
 (match_operand:V2DI 2 "register_operand" "")]
 "TARGET_SSE2"
 {
-rtx op1, op2, h1, l1;
+rtx op1 = gen_lowpart (V4SImode, operands[1]);
+rtx op2 = gen_lowpart (V4SImode, operands[2]);
-if (TARGET_SSE5)
+ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
-{
-ix86_expand_sse5_pack (operands);
-DONE;
-}
-op1 = gen_lowpart (V4SImode, operands[1]);
-op2 = gen_lowpart (V4SImode, operands[2]);
-h1 = gen_reg_rtx (V4SImode);
-l1 = gen_reg_rtx (V4SImode);
-emit_insn (gen_vec_interleave_highv4si (h1, op1, op2));
-emit_insn (gen_vec_interleave_lowv4si (l1, op1, op2));
-emit_insn (gen_vec_interleave_lowv4si (operands[0], l1, h1));
 DONE;
 })
-(define_expand "vec_interleave_highv16qi"
-[(set (match_operand:V16QI 0 "register_operand" "")
-(vec_select:V16QI
-(vec_concat:V32QI
-(match_operand:V16QI 1 "register_operand" "")
-(match_operand:V16QI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 8)  (const_int 24)
-(const_int 9)  (const_int 25)
-(const_int 10) (const_int 26)
-(const_int 11) (const_int 27)
-(const_int 12) (const_int 28)
-(const_int 13) (const_int 29)
-(const_int 14) (const_int 30)
-(const_int 15) (const_int 31)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpckhbw (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_lowv16qi"
-[(set (match_operand:V16QI 0 "register_operand" "")
-(vec_select:V16QI
-(vec_concat:V32QI
-(match_operand:V16QI 1 "register_operand" "")
-(match_operand:V16QI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0) (const_int 16)
-(const_int 1) (const_int 17)
-(const_int 2) (const_int 18)
-(const_int 3) (const_int 19)
-(const_int 4) (const_int 20)
-(const_int 5) (const_int 21)
-(const_int 6) (const_int 22)
-(const_int 7) (const_int 23)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpcklbw (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_highv8hi"
-[(set (match_operand:V8HI 0 "register_operand" "=")
-(vec_select:V8HI
-(vec_concat:V16HI
-(match_operand:V8HI 1 "register_operand" "")
-(match_operand:V8HI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 4) (const_int 12)
-(const_int 5) (const_int 13)
-(const_int 6) (const_int 14)
-(const_int 7) (const_int 15)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpckhwd (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_lowv8hi"
-[(set (match_operand:V8HI 0 "register_operand" "")
-(vec_select:V8HI
-(vec_concat:V16HI
-(match_operand:V8HI 1 "register_operand" "")
-(match_operand:V8HI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0) (const_int 8)
-(const_int 1) (const_int 9)
-(const_int 2) (const_int 10)
-(const_int 3) (const_int 11)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpcklwd (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_highv4si"
-[(set (match_operand:V4SI 0 "register_operand" "")
-(vec_select:V4SI
-(vec_concat:V8SI
-(match_operand:V4SI 1 "register_operand" "")
-(match_operand:V4SI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 2) (const_int 6)
-(const_int 3) (const_int 7)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpckhdq (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_lowv4si"
-[(set (match_operand:V4SI 0 "register_operand" "")
-(vec_select:V4SI
-(vec_concat:V8SI
-(match_operand:V4SI 1 "register_operand" "")
-(match_operand:V4SI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0) (const_int 4)
-(const_int 1) (const_int 5)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpckldq (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_highv2di"
-[(set (match_operand:V2DI 0 "register_operand" "")
-(vec_select:V2DI
-(vec_concat:V4DI
-(match_operand:V2DI 1 "register_operand" "")
-(match_operand:V2DI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 1)
-(const_int 3)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpckhqdq (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_lowv2di"
-[(set (match_operand:V2DI 0 "register_operand" "")
-(vec_select:V2DI
-(vec_concat:V4DI
-(match_operand:V2DI 1 "register_operand" "")
-(match_operand:V2DI 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0)
-(const_int 2)])))]
-"TARGET_SSE2"
-{
-emit_insn (gen_sse2_punpcklqdq (operands[0], operands[1], operands[2]));
-DONE;
-})
-(define_expand "vec_interleave_highv4sf"
-[(set (match_operand:V4SF 0 "register_operand" "")
-(vec_select:V4SF
-(vec_concat:V8SF
-(match_operand:V4SF 1 "register_operand" "")
-(match_operand:V4SF 2 "nonimmediate_operand" ""))
-(parallel [(const_int 2) (const_int 6)
-(const_int 3) (const_int 7)])))]
-"TARGET_SSE")
-(define_expand "vec_interleave_lowv4sf"
-[(set (match_operand:V4SF 0 "register_operand" "")
-(vec_select:V4SF
-(vec_concat:V8SF
-(match_operand:V4SF 1 "register_operand" "")
-(match_operand:V4SF 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0) (const_int 4)
-(const_int 1) (const_int 5)])))]
-"TARGET_SSE")
-(define_expand "vec_interleave_highv2df"
-[(set (match_operand:V2DF 0 "register_operand" "")
-(vec_select:V2DF
-(vec_concat:V4DF
-(match_operand:V2DF 1 "register_operand" "")
-(match_operand:V2DF 2 "nonimmediate_operand" ""))
-(parallel [(const_int 1)
-(const_int 3)])))]
-"TARGET_SSE2")
-(define_expand "vec_interleave_lowv2df"
-[(set (match_operand:V2DF 0 "register_operand" "")
-(vec_select:V2DF
-(vec_concat:V4DF
-(match_operand:V2DF 1 "register_operand" "")
-(match_operand:V2DF 2 "nonimmediate_operand" ""))
-(parallel [(const_int 0)
-(const_int 2)])))]
-"TARGET_SSE2")
 (define_insn "*avx_packsswb"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(vec_concat:V16QI
 	  (ss_truncate:V8QI
 "packuswb\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpckhbw"
+(define_insn "*avx_interleave_highv16qi"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(vec_select:V16QI
 	  (vec_concat:V32QI
 	    (match_operand:V16QI 1 "register_operand" "x")
 	    (match_operand:V16QI 2 "nonimmediate_operand" "xm"))
 "vpunpckhbw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpckhbw"
+(define_insn "vec_interleave_highv16qi"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(vec_select:V16QI
 	  (vec_concat:V32QI
 	    (match_operand:V16QI 1 "register_operand" "0")
 	    (match_operand:V16QI 2 "nonimmediate_operand" "xm"))
 "punpckhbw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpcklbw"
+(define_insn "*avx_interleave_lowv16qi"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(vec_select:V16QI
 	  (vec_concat:V32QI
 	    (match_operand:V16QI 1 "register_operand" "x")
 	    (match_operand:V16QI 2 "nonimmediate_operand" "xm"))
 "vpunpcklbw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpcklbw"
+(define_insn "vec_interleave_lowv16qi"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(vec_select:V16QI
 	  (vec_concat:V32QI
 	    (match_operand:V16QI 1 "register_operand" "0")
 	    (match_operand:V16QI 2 "nonimmediate_operand" "xm"))
 "punpcklbw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpckhwd"
+(define_insn "*avx_interleave_highv8hi"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_select:V8HI
 	  (vec_concat:V16HI
 	    (match_operand:V8HI 1 "register_operand" "x")
 	    (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
 "vpunpckhwd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpckhwd"
+(define_insn "vec_interleave_highv8hi"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_select:V8HI
 	  (vec_concat:V16HI
 	    (match_operand:V8HI 1 "register_operand" "0")
 	    (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
 "punpckhwd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpcklwd"
+(define_insn "*avx_interleave_lowv8hi"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_select:V8HI
 	  (vec_concat:V16HI
 	    (match_operand:V8HI 1 "register_operand" "x")
 	    (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
 "vpunpcklwd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpcklwd"
+(define_insn "vec_interleave_lowv8hi"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_select:V8HI
 	  (vec_concat:V16HI
 	    (match_operand:V8HI 1 "register_operand" "0")
 	    (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
 "punpcklwd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpckhdq"
+(define_insn "*avx_interleave_highv4si"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_select:V4SI
 	  (vec_concat:V8SI
 	    (match_operand:V4SI 1 "register_operand" "x")
 	    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
 "vpunpckhdq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpckhdq"
+(define_insn "vec_interleave_highv4si"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_select:V4SI
 	  (vec_concat:V8SI
 	    (match_operand:V4SI 1 "register_operand" "0")
 	    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
 "punpckhdq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_punpckldq"
+(define_insn "*avx_interleave_lowv4si"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_select:V4SI
 	  (vec_concat:V8SI
 	    (match_operand:V4SI 1 "register_operand" "x")
 	    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
 "vpunpckldq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
-(define_insn "sse2_punpckldq"
+(define_insn "vec_interleave_lowv4si"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_select:V4SI
 	  (vec_concat:V8SI
 	    (match_operand:V4SI 1 "register_operand" "0")
 	    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
 "punpckldq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
-(define_insn "*avx_pinsr<avxmodesuffixs>"
+(define_insn "*avx_pinsr<ssevecsize>"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(vec_merge:SSEMODE124
 	  (vec_duplicate:SSEMODE124
 	    (match_operand:<avxscalarmode> 2 "nonimmediate_operand" "rm"))
 	  (match_operand:SSEMODE124 1 "register_operand" "x")
 	  (match_operand:SI 3 "const_pow2_1_to_<pinsrbits>_operand" "n")))]
 "TARGET_AVX"
 {
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
 if (MEM_P (operands[2]))
-return "vpinsr<avxmodesuffixs>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+return "vpinsr<ssevecsize>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 else
-return "vpinsr<avxmodesuffixs>\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
+return "vpinsr<ssevecsize>\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
 }
 [(set_attr "type" "sselog")
+(set (attr "prefix_extra")
+(if_then_else (match_operand:V8HI 0 "register_operand" "")
+(const_string "0")
+(const_string "1")))
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pinsrb"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 else
 return "pinsrb\t{%3, %k2, %0|%0, %k2, %3}";
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*sse2_pinsrw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_merge:V8HI
 else
 return "pinsrw\t{%3, %k2, %0|%0, %k2, %3}";
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 ;; It must come before sse2_loadld since it is preferred.
 (define_insn "*sse4_1_pinsrd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
 return "pinsrd\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*avx_pinsrq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(vec_merge:V2DI
 {
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
 return "vpinsrq\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pinsrq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 {
 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
 return "pinsrq\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sselog")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_rex" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pextrb"
 [(set (match_operand:SI 0 "register_operand" "=r")
 	(zero_extend:SI
 	    (parallel [(match_operand:SI 2 "const_0_to_15_operand" "n")]))))]
 "TARGET_SSE4_1"
 "%vpextrb\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pextrb_memory"
 [(set (match_operand:QI 0 "memory_operand" "=m")
 	  (parallel [(match_operand:SI 2 "const_0_to_15_operand" "n")])))]
 "TARGET_SSE4_1"
 "%vpextrb\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse2_pextrw"
 [(set (match_operand:SI 0 "register_operand" "=r")
 	    (parallel [(match_operand:SI 2 "const_0_to_7_operand" "n")]))))]
 "TARGET_SSE2"
 "%vpextrw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pextrw_memory"
 [(set (match_operand:HI 0 "memory_operand" "=m")
 	  (parallel [(match_operand:SI 2 "const_0_to_7_operand" "n")])))]
 "TARGET_SSE4_1"
 "%vpextrw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse4_1_pextrd"
 [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
 	  (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n")])))]
 "TARGET_SSE4_1"
 "%vpextrd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 ;; It must come before *vec_extractv2di_1_sse since it is preferred.
 (define_insn "*sse4_1_pextrq"
 	  (match_operand:V2DI 1 "register_operand" "x")
 	  (parallel [(match_operand:SI 2 "const_0_to_1_operand" "n")])))]
 "TARGET_SSE4_1 && TARGET_64BIT"
 "%vpextrq\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_rex" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_expand "sse2_pshufd"
 [(match_operand:V4SI 0 "register_operand" "")
 return "%vpshufd\t{%2, %1, %0|%0, %1, %2}";
 }
 [(set_attr "type" "sselog1")
 (set_attr "prefix_data16" "1")
-(set_attr "prefix" "vex")
+(set_attr "prefix" "maybe_vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_expand "sse2_pshuflw"
 [(match_operand:V8HI 0 "register_operand" "")
 (match_operand:V8HI 1 "nonimmediate_operand" "")
 operands[2] = GEN_INT (mask);
 return "%vpshuflw\t{%2, %1, %0|%0, %1, %2}";
 }
 [(set_attr "type" "sselog")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix_rep" "1")
 (set_attr "prefix" "maybe_vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_expand "sse2_pshufhw"
 [(match_operand:V8HI 0 "register_operand" "")
 (match_operand:V8HI 1 "nonimmediate_operand" "")
 return "%vpshufhw\t{%2, %1, %0|%0, %1, %2}";
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_rep" "1")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix" "maybe_vex")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_expand "sse2_loadd"
 [(set (match_operand:V4SI 0 "register_operand" "")
 	(vec_merge:V4SI
 vmovhps\t{%1, %0|%0, %1}
 vpsrldq\t{$8, %1, %0|%0, %1, 8}
 vmovq\t{%H1, %0|%0, %H1}
 vmov{q}\t{%H1, %0|%0, %H1}"
 [(set_attr "type" "ssemov,sseishft,ssemov,imov")
+(set_attr "length_immediate" "*,1,*,*")
 (set_attr "memory" "*,none,*,*")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V2SF,TI,TI,DI")])
 (define_insn "*vec_extractv2di_1_rex64"
 movhps\t{%1, %0|%0, %1}
 psrldq\t{$8, %0|%0, 8}
 movq\t{%H1, %0|%0, %H1}
 mov{q}\t{%H1, %0|%0, %H1}"
 [(set_attr "type" "ssemov,sseishft,ssemov,imov")
+(set_attr "length_immediate" "*,1,*,*")
+(set_attr "atom_unit" "*,sishuf,*,*")
 (set_attr "memory" "*,none,*,*")
 (set_attr "mode" "V2SF,TI,TI,DI")])
 (define_insn "*vec_extractv2di_1_avx"
 [(set (match_operand:DI 0 "nonimmediate_operand" "=m,x,x")
 "@
 vmovhps\t{%1, %0|%0, %1}
 vpsrldq\t{$8, %1, %0|%0, %1, 8}
 vmovq\t{%H1, %0|%0, %H1}"
 [(set_attr "type" "ssemov,sseishft,ssemov")
+(set_attr "length_immediate" "*,1,*")
 (set_attr "memory" "*,none,*")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V2SF,TI,TI")])
 (define_insn "*vec_extractv2di_1_sse2"
 "@
 movhps\t{%1, %0|%0, %1}
 psrldq\t{$8, %0|%0, 8}
 movq\t{%H1, %0|%0, %H1}"
 [(set_attr "type" "ssemov,sseishft,ssemov")
+(set_attr "length_immediate" "*,1,*")
+(set_attr "atom_unit" "*,sishuf,*")
 (set_attr "memory" "*,none,*")
 (set_attr "mode" "V2SF,TI,TI")])
 ;; Not sure this is ever used, but it doesn't hurt to have it. -aoliva
 (define_insn "*vec_extractv2di_1_sse"
 movhlps\t{%1, %0|%0, %1}
 movlps\t{%H1, %0|%0, %H1}"
 [(set_attr "type" "ssemov")
 (set_attr "mode" "V2SF,V4SF,V2SF")])
+(define_insn "*vec_dupv4si_avx"
+[(set (match_operand:V4SI 0 "register_operand" "=x,x")
+	(vec_duplicate:V4SI
+	  (match_operand:SI 1 "register_operand" "x,m")))]
+"TARGET_AVX"
+"@
+vpshufd\t{$0, %1, %0|%0, %1, 0}
+vbroadcastss\t{%1, %0|%0, %1}"
+[(set_attr "type" "sselog1,ssemov")
+(set_attr "length_immediate" "1,0")
+(set_attr "prefix_extra" "0,1")
+(set_attr "prefix" "vex")
+(set_attr "mode" "TI,V4SF")])
 (define_insn "*vec_dupv4si"
 [(set (match_operand:V4SI 0 "register_operand" "=Y2,x")
 	(vec_duplicate:V4SI
 	  (match_operand:SI 1 "register_operand" " Y2,0")))]
 "TARGET_SSE"
 "@
 %vpshufd\t{$0, %1, %0|%0, %1, 0}
 shufps\t{$0, %0, %0|%0, %0, 0}"
 [(set_attr "type" "sselog1")
-(set_attr "prefix" "maybe_vex,orig")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI,V4SF")])
 (define_insn "*vec_dupv2di_avx"
-[(set (match_operand:V2DI 0 "register_operand" "=x")
+[(set (match_operand:V2DI 0 "register_operand"     "=x,x")
 	(vec_duplicate:V2DI
-	  (match_operand:DI 1 "register_operand" "x")))]
+	  (match_operand:DI 1 "nonimmediate_operand" " x,m")))]
 "TARGET_AVX"
-"vpunpcklqdq\t{%1, %1, %0|%0, %1, %1}"
+"@
+vpunpcklqdq\t{%1, %1, %0|%0, %1, %1}
+vmovddup\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog1")
 (set_attr "prefix" "vex")
-(set_attr "mode" "TI")])
+(set_attr "mode" "TI,DF")])
+(define_insn "*vec_dupv2di_sse3"
+[(set (match_operand:V2DI 0 "register_operand"     "=x,x")
+	(vec_duplicate:V2DI
+	  (match_operand:DI 1 "nonimmediate_operand" " 0,m")))]
+"TARGET_SSE3"
+"@
+punpcklqdq\t%0, %0
+movddup\t{%1, %0|%0, %1}"
+[(set_attr "type" "sselog1")
+(set_attr "mode" "TI,DF")])
 (define_insn "*vec_dupv2di"
 [(set (match_operand:V2DI 0 "register_operand" "=Y2,x")
 	(vec_duplicate:V2DI
 	  (match_operand:DI 1 "register_operand" " 0 ,0")))]
 vpunpckldq\t{%2, %1, %0|%0, %1, %2}
 vmovd\t{%1, %0|%0, %1}
 punpckldq\t{%2, %0|%0, %2}
 movd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog,sselog,ssemov,mmxcvt,mmxmov")
+(set_attr "prefix_extra" "1,*,*,*,*")
+(set_attr "length_immediate" "1,*,*,*,*")
 (set (attr "prefix")
 (if_then_else (eq_attr "alternative" "3,4")
 (const_string "orig")
 (const_string "vex")))
 (set_attr "mode" "TI,TI,TI,DI,DI")])
 movd\t{%1, %0|%0, %1}
 punpckldq\t{%2, %0|%0, %2}
 movd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sselog,sselog,ssemov,mmxcvt,mmxmov")
 (set_attr "prefix_extra" "1,*,*,*,*")
+(set_attr "length_immediate" "1,*,*,*,*")
 (set_attr "mode" "TI,TI,TI,DI,DI")])
 ;; ??? In theory we can match memory for the MMX alternative, but allowing
 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
 ;; alternatives pretty much forces the MMX alternative to be chosen.
 vmovq\t{%1, %0|%0, %1}
 movq2dq\t{%1, %0|%0, %1}
 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
 vmovhps\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog,ssemov,ssemov,ssemov,sselog,ssemov")
+(set_attr "prefix_extra" "1,*,*,*,*,*")
+(set_attr "length_immediate" "1,*,*,*,*,*")
 (set (attr "prefix")
 (if_then_else (eq_attr "alternative" "3")
 (const_string "orig")
 (const_string "vex")))
 (set_attr "mode" "TI,TI,TI,TI,TI,V2SF")])
 movq2dq\t{%1, %0|%0, %1}
 punpcklqdq\t{%2, %0|%0, %2}
 movlhps\t{%2, %0|%0, %2}
 movhps\t{%2, %0|%0, %2}"
 [(set_attr "type" "sselog,ssemov,ssemov,ssemov,sselog,ssemov,ssemov")
+(set_attr "prefix_rex" "1,*,1,*,*,*,*")
 (set_attr "prefix_extra" "1,*,*,*,*,*,*")
+(set_attr "length_immediate" "1,*,*,*,*,*,*")
 (set_attr "mode" "TI,TI,TI,TI,TI,V4SF,V2SF")])
 (define_insn "*vec_concatv2di_rex64_sse"
 [(set (match_operand:V2DI 0 "register_operand"     "=Y2 ,Yi,!Y2,Y2,x,x")
 	(vec_concat:V2DI
 movq2dq\t{%1, %0|%0, %1}
 punpcklqdq\t{%2, %0|%0, %2}
 movlhps\t{%2, %0|%0, %2}
 movhps\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssemov,ssemov,ssemov,sselog,ssemov,ssemov")
+(set_attr "prefix_rex" "*,1,*,*,*,*")
 (set_attr "mode" "TI,TI,TI,TI,V4SF,V2SF")])
 (define_expand "vec_unpacku_hi_v16qi"
 [(match_operand:V8HI 0 "register_operand" "")
 (match_operand:V16QI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, true);
 else
 ix86_expand_sse_unpack (operands, true, true);
 DONE;
 })
 (match_operand:V16QI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, true);
 else
 ix86_expand_sse_unpack (operands, false, true);
 DONE;
 })
 (match_operand:V16QI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, false);
 else
 ix86_expand_sse_unpack (operands, true, false);
 DONE;
 })
 (match_operand:V16QI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, false);
 else
 ix86_expand_sse_unpack (operands, false, false);
 DONE;
 })
 (match_operand:V8HI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, true);
 else
 ix86_expand_sse_unpack (operands, true, true);
 DONE;
 })
 (match_operand:V8HI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, true);
 else
 ix86_expand_sse_unpack (operands, false, true);
 DONE;
 })
 (match_operand:V8HI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, false);
 else
 ix86_expand_sse_unpack (operands, true, false);
 DONE;
 })
 (match_operand:V8HI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, false);
 else
 ix86_expand_sse_unpack (operands, false, false);
 DONE;
 })
 (match_operand:V4SI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, true);
 else
 ix86_expand_sse_unpack (operands, true, true);
 DONE;
 })
 (match_operand:V4SI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, true);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, true);
 else
 ix86_expand_sse_unpack (operands, false, true);
 DONE;
 })
 (match_operand:V4SI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, true, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, true, false);
 else
 ix86_expand_sse_unpack (operands, true, false);
 DONE;
 })
 (match_operand:V4SI 1 "register_operand" "")]
 "TARGET_SSE2"
 {
 if (TARGET_SSE4_1)
 ix86_expand_sse4_unpack (operands, false, false);
-else if (TARGET_SSE5)
-ix86_expand_sse5_unpack (operands, false, false);
 else
 ix86_expand_sse_unpack (operands, false, false);
 DONE;
 })
 		      (match_operand:V16QI 2 "nonimmediate_operand" "xm")]
 		     UNSPEC_PSADBW))]
 "TARGET_SSE2"
 "psadbw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "simul")
 (set_attr "prefix_data16" "1")
 (set_attr "mode" "TI")])
 (define_insn "avx_movmskp<avxmodesuffixf2c>256"
 [(set (match_operand:SI 0 "register_operand" "=r")
 	(unspec:SI
 	  [(match_operand:SSEMODEF2P 1 "register_operand" "x")]
 	  UNSPEC_MOVMSK))]
 "SSE_VEC_FLOAT_MODE_P (<MODE>mode)"
 "%vmovmskp<ssemodesuffixf2c>\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse2_pmovmskb"
 [(set (match_operand:SI 0 "register_operand" "=r")
 	(unspec:SI [(match_operand:V16QI 1 "register_operand" "x")]
 		   UNSPEC_MOVMSK))]
 "TARGET_SSE2"
 "%vpmovmskb\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "SI")])
 (define_expand "sse2_maskmovdqu"
 		       (mem:V16QI (match_dup 0))]
 		      UNSPEC_MASKMOV))]
 "TARGET_SSE2 && !TARGET_64BIT"
 ;; @@@ check ordering of operands in intel/nonintel syntax
 "%vmaskmovdqu\t{%2, %1|%1, %2}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix_data16" "1")
+;; The implicit %rdi operand confuses default length_vex computation.
+(set_attr "length_vex" "3")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*sse2_maskmovdqu_rex64"
 [(set (mem:V16QI (match_operand:DI 0 "register_operand" "D"))
 		       (mem:V16QI (match_dup 0))]
 		      UNSPEC_MASKMOV))]
 "TARGET_SSE2 && TARGET_64BIT"
 ;; @@@ check ordering of operands in intel/nonintel syntax
 "%vmaskmovdqu\t{%2, %1|%1, %2}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix_data16" "1")
+;; The implicit %rdi operand confuses default length_vex computation.
+(set (attr "length_vex")
+(symbol_ref ("REGNO (operands[2]) >= FIRST_REX_SSE_REG ? 3 + 1 : 2 + 1")))
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "sse_ldmxcsr"
 [(unspec_volatile [(match_operand:SI 0 "memory_operand" "m")]
 		    UNSPECV_LDMXCSR)]
 "TARGET_SSE"
 "%vldmxcsr\t%0"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "mxcsr")
 (set_attr "prefix" "maybe_vex")
 (set_attr "memory" "load")])
 (define_insn "sse_stmxcsr"
 [(set (match_operand:SI 0 "memory_operand" "=m")
 	(unspec_volatile:SI [(const_int 0)] UNSPECV_STMXCSR))]
 "TARGET_SSE"
 "%vstmxcsr\t%0"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "mxcsr")
 (set_attr "prefix" "maybe_vex")
 (set_attr "memory" "store")])
 (define_expand "sse_sfence"
 [(set (match_dup 0)
 [(set (match_operand:BLK 0 "" "")
 	(unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
 "TARGET_SSE || TARGET_3DNOW_A"
 "sfence"
 [(set_attr "type" "sse")
+(set_attr "length_address" "0")
+(set_attr "atom_sse_attr" "fence")
 (set_attr "memory" "unknown")])
 (define_insn "sse2_clflush"
 [(unspec_volatile [(match_operand 0 "address_operand" "p")]
 		    UNSPECV_CLFLUSH)]
 "TARGET_SSE2"
 "clflush\t%a0"
 [(set_attr "type" "sse")
+(set_attr "atom_sse_attr" "fence")
 (set_attr "memory" "unknown")])
 (define_expand "sse2_mfence"
 [(set (match_dup 0)
 	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
 [(set (match_operand:BLK 0 "" "")
 	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
 "TARGET_64BIT || TARGET_SSE2"
 "mfence"
 [(set_attr "type" "sse")
+(set_attr "length_address" "0")
+(set_attr "atom_sse_attr" "fence")
 (set_attr "memory" "unknown")])
 (define_expand "sse2_lfence"
 [(set (match_dup 0)
 	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
 [(set (match_operand:BLK 0 "" "")
 	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
 "TARGET_SSE2"
 "lfence"
 [(set_attr "type" "sse")
+(set_attr "length_address" "0")
+(set_attr "atom_sse_attr" "lfence")
 (set_attr "memory" "unknown")])
 (define_insn "sse3_mwait"
 [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
 		     (match_operand:SI 1 "register_operand" "c")]
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_AVX"
 "vphaddw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phaddwv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_SSSE3"
 "phaddw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phaddwv4hi3"
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phaddw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_phadddv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_concat:V4SI
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_AVX"
 "vphaddd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phadddv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phaddd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phadddv2si3"
 	      (parallel [(const_int 0)]))
 	    (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
 "TARGET_SSSE3"
 "phaddd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_phaddswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_concat:V8HI
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_AVX"
 "vphaddsw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phaddswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_SSSE3"
 "phaddsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phaddswv4hi3"
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phaddsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_phsubwv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_concat:V8HI
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_AVX"
 "vphsubw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubwv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_SSSE3"
 "phsubw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubwv4hi3"
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phsubw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_phsubdv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(vec_concat:V4SI
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_AVX"
 "vphsubd\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubdv4si3"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phsubd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubdv2si3"
 	      (parallel [(const_int 0)]))
 	    (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
 "TARGET_SSSE3"
 "phsubd\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_phsubswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_concat:V8HI
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_AVX"
 "vphsubsw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 		(vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
 		(vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
 "TARGET_SSSE3"
 "phsubsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "complex")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_phsubswv4hi3"
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
 	      (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
 "TARGET_SSSE3"
 "phsubsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "complex")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_pmaddubsw128"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(ss_plus:V8HI
 			   (const_int 13)
 			   (const_int 15)]))))))]
 "TARGET_AVX"
 "vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_pmaddubsw128"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 			   (const_int 13)
 			   (const_int 15)]))))))]
 "TARGET_SSSE3"
 "pmaddubsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
+(set_attr "atom_unit" "simul")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_pmaddubsw"
 			   (const_int 5)
 			   (const_int 7)]))))))]
 "TARGET_SSSE3"
 "pmaddubsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseiadd")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "simul")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_expand "ssse3_pmulhrswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "")
 	(truncate:V8HI
 				  (const_int 1) (const_int 1)]))
 	    (const_int 1))))]
 "TARGET_AVX && ix86_binary_operator_ok (MULT, V8HImode, operands)"
 "vpmulhrsw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseimul")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "*ssse3_pmulhrswv8hi3"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	    (const_int 1))))]
 "TARGET_SSSE3 && ix86_binary_operator_ok (MULT, V4HImode, operands)"
 "pmulhrsw\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseimul")
 (set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_pshufbv16qi3"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(unspec:V16QI [(match_operand:V16QI 1 "register_operand" "x")
 		       (match_operand:V16QI 2 "nonimmediate_operand" "xm")]
 		      UNSPEC_PSHUFB))]
 "TARGET_AVX"
 "vpshufb\t{%2, %1, %0|%0, %1, %2}";
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_pshufbv16qi3"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 		     UNSPEC_PSHUFB))]
 "TARGET_SSSE3"
 "pshufb\t{%2, %0|%0, %2}";
 [(set_attr "type" "sselog1")
 (set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_psign<mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(unspec:SSEMODE124
 	   (match_operand:SSEMODE124 2 "nonimmediate_operand" "xm")]
 	  UNSPEC_PSIGN))]
 "TARGET_AVX"
 "vpsign<ssevecsize>\t{%2, %1, %0|%0, %1, %2}";
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_psign<mode>3"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	  UNSPEC_PSIGN))]
 "TARGET_SSSE3"
 "psign<mmxvecsize>\t{%2, %0|%0, %2}";
 [(set_attr "type" "sselog1")
 (set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "*avx_palignrti"
 [(set (match_operand:TI 0 "register_operand" "=x")
 	(unspec:TI [(match_operand:TI 1 "register_operand" "x")
 {
 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
 return "vpalignr\t{%3, %2, %1, %0|%0, %1, %2, %3}";
 }
 [(set_attr "type" "sseishft")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_palignrti"
 [(set (match_operand:TI 0 "register_operand" "=x")
 {
 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
 return "palignr\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sseishft")
+(set_attr "atom_unit" "sishuf")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "ssse3_palignrdi"
 [(set (match_operand:DI 0 "register_operand" "=y")
 	(unspec:DI [(match_operand:DI 1 "register_operand" "0")
 {
 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
 return "palignr\t{%3, %2, %0|%0, %2, %3}";
 }
 [(set_attr "type" "sseishft")
-(set_attr "prefix_extra" "1")
+(set_attr "atom_unit" "sishuf")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 (define_insn "abs<mode>2"
 [(set (match_operand:SSEMODE124 0 "register_operand" "=x")
 	(abs:SSEMODE124 (match_operand:SSEMODE124 1 "nonimmediate_operand" "xm")))]
 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
 	(abs:MMXMODEI (match_operand:MMXMODEI 1 "nonimmediate_operand" "ym")))]
 "TARGET_SSSE3"
 "pabs<mmxvecsize>\t{%1, %0|%0, %1}";
 [(set_attr "type" "sselog1")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_rep" "0")
+(set_attr "prefix_extra" "1")
+(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
 (set_attr "mode" "DI")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; AMD SSE4A instructions
 UNSPEC_EXTRQI))]
 "TARGET_SSE4A"
 "extrq\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "sse")
 (set_attr "prefix_data16" "1")
+(set_attr "length_immediate" "2")
 (set_attr "mode" "TI")])
 (define_insn "sse4a_extrq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
 (match_operand 4 "const_int_operand" "")]
 UNSPEC_INSERTQI))]
 "TARGET_SSE4A"
 "insertq\t{%4, %3, %2, %0|%0, %2, %3, %4}"
 [(set_attr "type" "sseins")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix_rep" "1")
+(set_attr "length_immediate" "2")
 (set_attr "mode" "TI")])
 (define_insn "sse4a_insertq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
 	      (match_operand:V2DI 2 "register_operand" "x")]
 	     UNSPEC_INSERTQ))]
 "TARGET_SSE4A"
 "insertq\t{%2, %0|%0, %2}"
 [(set_attr "type" "sseins")
+(set_attr "prefix_data16" "0")
 (set_attr "prefix_rep" "1")
 (set_attr "mode" "TI")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 	  (match_operand:AVXMODEF2P 1 "register_operand" "x")
 	  (match_operand:SI 3 "const_0_to_<blendbits>_operand" "n")))]
 "TARGET_AVX"
 "vblendp<avxmodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemov")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "avx_blendvp<avxmodesuffixf2c><avxmodesuffix>"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
 	   (match_operand:AVXMODEF2P 3 "register_operand" "x")]
 	  UNSPEC_BLENDV))]
 "TARGET_AVX"
 "vblendvp<avxmodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemov")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "sse4_1_blendp<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	  (match_operand:SSEMODEF2P 1 "register_operand" "0")
 	  (match_operand:SI 3 "const_0_to_<blendbits>_operand" "n")))]
 "TARGET_SSE4_1"
 "blendp<ssemodesuffixf2c>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssemov")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse4_1_blendvp<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "reg_not_xmm0_operand" "=x")
 	(unspec:SSEMODEF2P
 	   (match_operand:SSEMODEF2P 3 "register_operand" "Yz")]
 	  UNSPEC_BLENDV))]
 "TARGET_SSE4_1"
 "blendvp<ssemodesuffixf2c>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssemov")
+(set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_dpp<avxmodesuffixf2c><avxmodesuffix>"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
 	  UNSPEC_DP))]
 "TARGET_AVX"
 "vdpp<avxmodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemul")
 (set_attr "prefix" "vex")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<avxvecmode>")])
 (define_insn "sse4_1_dpp<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	(unspec:SSEMODEF2P
 	   (match_operand:SI 3 "const_0_to_255_operand" "n")]
 	  UNSPEC_DP))]
 "TARGET_SSE4_1"
 "dpp<ssemodesuffixf2c>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssemul")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse4_1_movntdqa"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "m")]
 		     UNSPEC_MOVNTDQA))]
 "TARGET_SSE4_1"
 "%vmovntdqa\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
+[(set_attr "type" "ssemov")
 (set_attr "prefix_extra" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*avx_mpsadbw"
 		      UNSPEC_MPSADBW))]
 "TARGET_AVX"
 "vmpsadbw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sselog1")
 (set_attr "prefix" "vex")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "sse4_1_mpsadbw"
 [(set (match_operand:V16QI 0 "register_operand" "=x")
 	(unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
 		      UNSPEC_MPSADBW))]
 "TARGET_SSE4_1"
 "mpsadbw\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "sselog1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "*avx_packusdw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_concat:V8HI
 	  (us_truncate:V4HI
 	    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))))]
 "TARGET_AVX"
 "vpackusdw\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "sse4_1_packusdw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 		       (match_operand:V16QI 3 "register_operand" "x")]
 		      UNSPEC_BLENDV))]
 "TARGET_AVX"
 "vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemov")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "sse4_1_pblendvb"
 [(set (match_operand:V16QI 0 "reg_not_xmm0_operand" "=x")
 	  (match_operand:SI 3 "const_0_to_255_operand" "n")))]
 "TARGET_AVX"
 "vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssemov")
 (set_attr "prefix" "vex")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "sse4_1_pblendw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(vec_merge:V8HI
 	  (match_operand:SI 3 "const_0_to_255_operand" "n")))]
 "TARGET_SSE4_1"
 "pblendw\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssemov")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_insn "sse4_1_phminposuw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(unspec:V8HI [(match_operand:V8HI 1 "nonimmediate_operand" "xm")]
 		    (match_operand:AVXMODEF2P 1 "nonimmediate_operand" "xm")]
 		   UNSPEC_VTESTP))]
 "TARGET_AVX"
 "vtestp<avxmodesuffixf2c>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecomi")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 ;; ptest is very similar to comiss and ucomiss when setting FLAGS_REG.
 ;; But it is not a really compare instruction.
 		    (match_operand:V4DI 1 "nonimmediate_operand" "xm")]
 		   UNSPEC_PTEST))]
 "TARGET_AVX"
 "vptest\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecomi")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "OI")])
 (define_insn "sse4_1_ptest"
 [(set (reg:CC FLAGS_REG)
 	   (match_operand:SI 2 "const_0_to_15_operand" "n")]
 	  UNSPEC_ROUND))]
 "TARGET_AVX"
 "vroundp<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "ssecvt")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse4_1_roundp<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	   (match_operand:SI 2 "const_0_to_15_operand" "n")]
 	  UNSPEC_ROUND))]
 "TARGET_ROUND"
 "%vroundp<ssemodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "ssecvt")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "*avx_rounds<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	  (match_operand:SSEMODEF2P 1 "register_operand" "x")
 	  (const_int 1)))]
 "TARGET_AVX"
 "vrounds<ssemodesuffixf2c>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "ssecvt")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "sse4_1_rounds<ssemodesuffixf2c>"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	  (match_operand:SSEMODEF2P 1 "register_operand" "0")
 	  (const_int 1)))]
 "TARGET_ROUND"
 "rounds<ssemodesuffixf2c>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssecvt")
-(set_attr "prefix_extra" "1")
+(set_attr "prefix_data16" "1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "<MODE>")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Intel SSE4.2 string/text processing instructions
 	   (match_dup 4)
 	   (match_dup 5)
 	   (match_dup 6)]
 	  UNSPEC_PCMPESTR))]
 "TARGET_SSE4_2
-&& !(reload_completed || reload_in_progress)"
+&& can_create_pseudo_p ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
 DONE;
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpestri"
 [(set (match_operand:SI 0 "register_operand" "=c,c")
 "%vpcmpestri\t{%5, %3, %1|%1, %3, %5}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
 (set_attr "prefix" "maybe_vex")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpestrm"
 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
 "TARGET_SSE4_2"
 "%vpcmpestrm\t{%5, %3, %1|%1, %3, %5}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpestr_cconly"
 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}
 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,load,none,load")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn_and_split "sse4_2_pcmpistr"
 	  [(match_dup 2)
 	   (match_dup 3)
 	   (match_dup 4)]
 	  UNSPEC_PCMPISTR))]
 "TARGET_SSE4_2
-&& !(reload_completed || reload_in_progress)"
+&& can_create_pseudo_p ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
 DONE;
 }
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpistri"
 [(set (match_operand:SI 0 "register_operand" "=c,c")
 "TARGET_SSE4_2"
 "%vpcmpistri\t{%3, %2, %1|%1, %2, %3}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpistrm"
 "TARGET_SSE4_2"
 "%vpcmpistrm\t{%3, %2, %1|%1, %2, %3}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "memory" "none,load")
 (set_attr "mode" "TI")])
 (define_insn "sse4_2_pcmpistr_cconly"
 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}
 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}"
 [(set_attr "type" "sselog")
 (set_attr "prefix_data16" "1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "memory" "none,load,none,load")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
-;; SSE5 instructions
+;; XOP instructions
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; SSE5 parallel integer multiply/add instructions.
+;; XOP parallel integer multiply/add instructions.
-;; Note the instruction does not allow the value being added to be a memory
+;; Note the XOP multiply/add instructions
-;; operation.  However by pretending via the nonimmediate_operand predicate
+;;     a[i] = b[i] * c[i] + d[i];
-;; that it does and splitting it later allows the following to be recognized:
+;; do not allow the value being added to be a memory operation.
-;;	a[i] = b[i] * c[i] + d[i];
+(define_insn "xop_pmacsww"
-(define_insn "sse5_pmacsww"
+[(set (match_operand:V8HI 0 "register_operand" "=x")
-[(set (match_operand:V8HI 0 "register_operand" "=x,x,x")
 (plus:V8HI
 	 (mult:V8HI
-	  (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,xm")
+	  (match_operand:V8HI 1 "nonimmediate_operand" "%x")
-	  (match_operand:V8HI 2 "nonimmediate_operand" "x,xm,x"))
+	  (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
-	 (match_operand:V8HI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V8HI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 2, true)"
+"TARGET_XOP"
-"@
+"vpmacsww\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacsww\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsww\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsww\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-;; Split pmacsww with two memory operands into a load and the pmacsww.
+(define_insn "xop_pmacssww"
-(define_split
+[(set (match_operand:V8HI 0 "register_operand" "=x")
-[(set (match_operand:V8HI 0 "register_operand" "")
-	(plus:V8HI
-	 (mult:V8HI (match_operand:V8HI 1 "nonimmediate_operand" "")
-		    (match_operand:V8HI 2 "nonimmediate_operand" ""))
-	 (match_operand:V8HI 3 "nonimmediate_operand" "")))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)
-&& ix86_sse5_valid_op_p (operands, insn, 4, false, 2, true)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, V8HImode);
-emit_insn (gen_sse5_pmacsww (operands[0], operands[1], operands[2],
-			       operands[3]));
-DONE;
-})
-(define_insn "sse5_pmacssww"
-[(set (match_operand:V8HI 0 "register_operand" "=x,x,x")
 (ss_plus:V8HI
-	 (mult:V8HI (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,m")
+	 (mult:V8HI (match_operand:V8HI 1 "nonimmediate_operand" "%x")
-		    (match_operand:V8HI 2 "nonimmediate_operand" "x,m,x"))
+		    (match_operand:V8HI 2 "nonimmediate_operand" "xm"))
-	 (match_operand:V8HI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V8HI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacssww\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacssww\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssww\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssww\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-;; Note the instruction does not allow the value being added to be a memory
+(define_insn "xop_pmacsdd"
-;; operation.  However by pretending via the nonimmediate_operand predicate
+[(set (match_operand:V4SI 0 "register_operand" "=x")
-;; that it does and splitting it later allows the following to be recognized:
-;;	a[i] = b[i] * c[i] + d[i];
-(define_insn "sse5_pmacsdd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
 (plus:V4SI
 	 (mult:V4SI
-	  (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	  (match_operand:V4SI 1 "nonimmediate_operand" "%x")
-	  (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x"))
+	  (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 2, true)"
+"TARGET_XOP"
-"@
+"vpmacsdd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacsdd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-;; Split pmacsdd with two memory operands into a load and the pmacsdd.
+(define_insn "xop_pmacssdd"
-(define_split
+[(set (match_operand:V4SI 0 "register_operand" "=x")
-[(set (match_operand:V4SI 0 "register_operand" "")
-	(plus:V4SI
-	 (mult:V4SI (match_operand:V4SI 1 "nonimmediate_operand" "")
-		    (match_operand:V4SI 2 "nonimmediate_operand" ""))
-	 (match_operand:V4SI 3 "nonimmediate_operand" "")))]
-"TARGET_SSE5
-&& !ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)
-&& ix86_sse5_valid_op_p (operands, insn, 4, false, 2, true)
-&& !reg_mentioned_p (operands[0], operands[1])
-&& !reg_mentioned_p (operands[0], operands[2])
-&& !reg_mentioned_p (operands[0], operands[3])"
-[(const_int 0)]
-{
-ix86_expand_sse5_multiple_memory (operands, 4, V4SImode);
-emit_insn (gen_sse5_pmacsdd (operands[0], operands[1], operands[2],
-			       operands[3]));
-DONE;
-})
-(define_insn "sse5_pmacssdd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
 (ss_plus:V4SI
-	 (mult:V4SI (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	 (mult:V4SI (match_operand:V4SI 1 "nonimmediate_operand" "%x")
-		    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x"))
+		    (match_operand:V4SI 2 "nonimmediate_operand" "xm"))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacssdd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacssdd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmacssdql"
+(define_insn "xop_pmacssdql"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V2DI 0 "register_operand" "=x")
 	(ss_plus:V2DI
 	 (mult:V2DI
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V4SI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 1)
 		       (const_int 3)])))
-	   (vec_select:V2SI
+	  (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
+	   (match_operand:V4SI 2 "nonimmediate_operand" "xm")
-	    (parallel [(const_int 1)
+	   (parallel [(const_int 1)
-		       (const_int 3)])))
+		      (const_int 3)])))
-	 (match_operand:V2DI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacssdql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacssdql\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdql\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdql\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmacssdqh"
+(define_insn "xop_pmacssdqh"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V2DI 0 "register_operand" "=x")
 	(ss_plus:V2DI
 	 (mult:V2DI
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V4SI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 0)
 		       (const_int 2)])))
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
+	    (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	    (parallel [(const_int 0)
 		       (const_int 2)]))))
-	 (match_operand:V2DI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacssdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacssdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacssdqh\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmacsdql"
+(define_insn "xop_pmacsdql"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (mult:V2DI
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V4SI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 1)
 		       (const_int 3)])))
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
+	    (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	    (parallel [(const_int 1)
 		       (const_int 3)]))))
-	 (match_operand:V2DI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacsdql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacsdql\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdql\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdql\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn_and_split "*sse5_pmacsdql_mem"
+;; We don't have a straight 32-bit parallel multiply and extend on XOP, so
-[(set (match_operand:V2DI 0 "register_operand" "=&x,&x,&x")
-	(plus:V2DI
-	 (mult:V2DI
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
-	    (parallel [(const_int 1)
-		       (const_int 3)])))
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
-	    (parallel [(const_int 1)
-		       (const_int 3)]))))
-	 (match_operand:V2DI 3 "memory_operand" "m,m,m")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, -1, true)"
-"#"
-"&& (reload_completed
-|| (!reg_mentioned_p (operands[0], operands[1])
-	   && !reg_mentioned_p (operands[0], operands[2])))"
-[(set (match_dup 0)
-	(match_dup 3))
-(set (match_dup 0)
-	(plus:V2DI
-	 (mult:V2DI
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_dup 1)
-	    (parallel [(const_int 1)
-		       (const_int 3)])))
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_dup 2)
-	    (parallel [(const_int 1)
-		       (const_int 3)]))))
-	 (match_dup 0)))])
-;; We don't have a straight 32-bit parallel multiply and extend on SSE5, so
 ;; fake it with a multiply/add.  In general, we expect the define_split to
 ;; occur before register allocation, so we have to handle the corner case where
 ;; the target is the same as operands 1/2
-(define_insn_and_split "sse5_mulv2div2di3_low"
+(define_insn_and_split "xop_mulv2div2di3_low"
 [(set (match_operand:V2DI 0 "register_operand" "=&x")
 	(mult:V2DI
 	  (sign_extend:V2DI
 	    (vec_select:V2SI
-	      (match_operand:V4SI 1 "nonimmediate_operand" "%x")
+	      (match_operand:V4SI 1 "register_operand" "%x")
 	      (parallel [(const_int 1)
 			 (const_int 3)])))
 	  (sign_extend:V2DI
 	    (vec_select:V2SI
 	      (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	      (parallel [(const_int 1)
 			 (const_int 3)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
 "#"
-"&& (reload_completed
+"&& reload_completed"
-|| (!reg_mentioned_p (operands[0], operands[1])
-	   && !reg_mentioned_p (operands[0], operands[2])))"
 [(set (match_dup 0)
 	(match_dup 3))
 (set (match_dup 0)
 	(plus:V2DI
 	 (mult:V2DI
 operands[3] = CONST0_RTX (V2DImode);
 }
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmacsdqh"
+(define_insn "xop_pmacsdqh"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (mult:V2DI
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V4SI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 0)
 		       (const_int 2)])))
 	  (sign_extend:V2DI
 	   (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
+	    (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	    (parallel [(const_int 0)
 		       (const_int 2)]))))
-	 (match_operand:V2DI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacsdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacsdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsdqh\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn_and_split "*sse5_pmacsdqh_mem"
+;; We don't have a straight 32-bit parallel multiply and extend on XOP, so
-[(set (match_operand:V2DI 0 "register_operand" "=&x,&x,&x")
-	(plus:V2DI
-	 (mult:V2DI
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_operand:V4SI 1 "nonimmediate_operand" "%x,x,m")
-	    (parallel [(const_int 0)
-		       (const_int 2)])))
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_operand:V4SI 2 "nonimmediate_operand" "x,m,x")
-	    (parallel [(const_int 0)
-		       (const_int 2)]))))
-	 (match_operand:V2DI 3 "memory_operand" "m,m,m")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, -1, true)"
-"#"
-"&& (reload_completed
-|| (!reg_mentioned_p (operands[0], operands[1])
-	   && !reg_mentioned_p (operands[0], operands[2])))"
-[(set (match_dup 0)
-	(match_dup 3))
-(set (match_dup 0)
-	(plus:V2DI
-	 (mult:V2DI
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_dup 1)
-	    (parallel [(const_int 0)
-		       (const_int 2)])))
-	  (sign_extend:V2DI
-	   (vec_select:V2SI
-	    (match_dup 2)
-	    (parallel [(const_int 0)
-		       (const_int 2)]))))
-	 (match_dup 0)))])
-;; We don't have a straight 32-bit parallel multiply and extend on SSE5, so
 ;; fake it with a multiply/add.  In general, we expect the define_split to
 ;; occur before register allocation, so we have to handle the corner case where
 ;; the target is the same as either operands[1] or operands[2]
-(define_insn_and_split "sse5_mulv2div2di3_high"
+(define_insn_and_split "xop_mulv2div2di3_high"
 [(set (match_operand:V2DI 0 "register_operand" "=&x")
 	(mult:V2DI
 	  (sign_extend:V2DI
 	    (vec_select:V2SI
-	      (match_operand:V4SI 1 "nonimmediate_operand" "%x")
+	      (match_operand:V4SI 1 "register_operand" "%x")
 	      (parallel [(const_int 0)
 			 (const_int 2)])))
 	  (sign_extend:V2DI
 	    (vec_select:V2SI
 	      (match_operand:V4SI 2 "nonimmediate_operand" "xm")
 	      (parallel [(const_int 0)
 			 (const_int 2)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
 "#"
-"&& (reload_completed
+"&& reload_completed"
-|| (!reg_mentioned_p (operands[0], operands[1])
-	   && !reg_mentioned_p (operands[0], operands[2])))"
 [(set (match_dup 0)
 	(match_dup 3))
 (set (match_dup 0)
 	(plus:V2DI
 	 (mult:V2DI
 operands[3] = CONST0_RTX (V2DImode);
 }
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-;; SSE5 parallel integer multiply/add instructions for the intrinisics
+;; XOP parallel integer multiply/add instructions for the intrinisics
-(define_insn "sse5_pmacsswd"
+(define_insn "xop_pmacsswd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V4SI 0 "register_operand" "=x")
 	(ss_plus:V4SI
 	 (mult:V4SI
 	  (sign_extend:V4SI
 	   (vec_select:V4HI
-	    (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V8HI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 1)
 		       (const_int 3)
 		       (const_int 5)
 		       (const_int 7)])))
 	  (sign_extend:V4SI
 	   (vec_select:V4HI
-	    (match_operand:V8HI 2 "nonimmediate_operand" "x,m,x")
+	    (match_operand:V8HI 2 "nonimmediate_operand" "xm")
 	    (parallel [(const_int 1)
 		       (const_int 3)
 		       (const_int 5)
 		       (const_int 7)]))))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacsswd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmacswd"
+(define_insn "xop_pmacswd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (mult:V4SI
 	  (sign_extend:V4SI
 	   (vec_select:V4HI
-	    (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,m")
+	    (match_operand:V8HI 1 "nonimmediate_operand" "%x")
 	    (parallel [(const_int 1)
 		       (const_int 3)
 		       (const_int 5)
 		       (const_int 7)])))
 	  (sign_extend:V4SI
 	   (vec_select:V4HI
-	    (match_operand:V8HI 2 "nonimmediate_operand" "x,m,x")
+	    (match_operand:V8HI 2 "nonimmediate_operand" "xm")
 	    (parallel [(const_int 1)
 		       (const_int 3)
 		       (const_int 5)
 		       (const_int 7)]))))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmacswd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmacswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmacswd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmadcsswd"
+(define_insn "xop_pmadcsswd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V4SI 0 "register_operand" "=x")
 	(ss_plus:V4SI
 	 (plus:V4SI
 	  (mult:V4SI
 	   (sign_extend:V4SI
 	    (vec_select:V4HI
-	     (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,m")
+	     (match_operand:V8HI 1 "nonimmediate_operand" "%x")
 	     (parallel [(const_int 0)
 			(const_int 2)
 			(const_int 4)
 			(const_int 6)])))
 	   (sign_extend:V4SI
 	    (vec_select:V4HI
-	     (match_operand:V8HI 2 "nonimmediate_operand" "x,m,x")
+	     (match_operand:V8HI 2 "nonimmediate_operand" "xm")
 	     (parallel [(const_int 0)
 			(const_int 2)
 			(const_int 4)
 			(const_int 6)]))))
 	  (mult:V4SI
 	     (match_dup 2)
 	     (parallel [(const_int 1)
 			(const_int 3)
 			(const_int 5)
 			(const_int 7)])))))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmadcsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmadcsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmadcsswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmadcsswd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pmadcswd"
+(define_insn "xop_pmadcswd"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
+[(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (plus:V4SI
 	  (mult:V4SI
 	   (sign_extend:V4SI
 	    (vec_select:V4HI
-	     (match_operand:V8HI 1 "nonimmediate_operand" "%x,x,m")
+	     (match_operand:V8HI 1 "nonimmediate_operand" "%x")
 	     (parallel [(const_int 0)
 			(const_int 2)
 			(const_int 4)
 			(const_int 6)])))
 	   (sign_extend:V4SI
 	    (vec_select:V4HI
-	     (match_operand:V8HI 2 "nonimmediate_operand" "x,m,x")
+	     (match_operand:V8HI 2 "nonimmediate_operand" "xm")
 	     (parallel [(const_int 0)
 			(const_int 2)
 			(const_int 4)
 			(const_int 6)]))))
 	  (mult:V4SI
 	     (match_dup 2)
 	     (parallel [(const_int 1)
 			(const_int 3)
 			(const_int 5)
 			(const_int 7)])))))
-	 (match_operand:V4SI 3 "register_operand" "0,0,0")))]
+	 (match_operand:V4SI 3 "nonimmediate_operand" "x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, false, 1, true)"
+"TARGET_XOP"
-"@
+"vpmadcswd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pmadcswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmadcswd\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pmadcswd\t{%3, %1, %2, %0|%0, %2, %1, %3}"
 [(set_attr "type" "ssemuladd")
 (set_attr "mode" "TI")])
-;; SSE5 parallel XMM conditional moves
+;; XOP parallel XMM conditional moves
-(define_insn "sse5_pcmov_<mode>"
+(define_insn "xop_pcmov_<mode>"
-[(set (match_operand:SSEMODE 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:SSEMODE 0 "register_operand" "=x,x")
 	(if_then_else:SSEMODE
-	  (match_operand:SSEMODE 3 "nonimmediate_operand" "0,0,xm,x")
+	  (match_operand:SSEMODE 3 "nonimmediate_operand" "x,m")
-	  (match_operand:SSEMODE 1 "vector_move_operand" "x,xm,0,0")
+	  (match_operand:SSEMODE 1 "vector_move_operand" "x,x")
-	  (match_operand:SSEMODE 2 "vector_move_operand" "xm,x,x,xm")))]
+	  (match_operand:SSEMODE 2 "vector_move_operand" "xm,x")))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_XOP"
-"@
+"vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-pcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}
-pcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sse4arg")])
-;; SSE5 horizontal add/subtract instructions
+(define_insn "xop_pcmov_<mode>256"
-(define_insn "sse5_phaddbw"
+[(set (match_operand:AVX256MODE 0 "register_operand" "=x,x")
+	(if_then_else:AVX256MODE
+	  (match_operand:AVX256MODE 3 "nonimmediate_operand" "x,m")
+	  (match_operand:AVX256MODE 1 "vector_move_operand" "x,x")
+	  (match_operand:AVX256MODE 2 "vector_move_operand" "xm,x")))]
+"TARGET_XOP"
+"vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+[(set_attr "type" "sse4arg")])
+;; XOP horizontal add/subtract instructions
+(define_insn "xop_phaddbw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(plus:V8HI
 	 (sign_extend:V8HI
 	  (vec_select:V8QI
 	   (match_operand:V16QI 1 "nonimmediate_operand" "xm")
 		      (const_int 7)
 		      (const_int 9)
 		      (const_int 11)
 		      (const_int 13)
 		      (const_int 15)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddbw\t{%1, %0|%0, %1}"
+"vphaddbw\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddbd"
+(define_insn "xop_phaddbd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (plus:V4SI
 	  (sign_extend:V4SI
 	   (vec_select:V4QI
 	    (match_dup 1)
 	    (parallel [(const_int 3)
 		       (const_int 7)
 		       (const_int 11)
 		       (const_int 15)]))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddbd\t{%1, %0|%0, %1}"
+"vphaddbd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddbq"
+(define_insn "xop_phaddbq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (plus:V2DI
 	  (plus:V2DI
 	   (sign_extend:V2DI
 	   (sign_extend:V2DI
 	    (vec_select:V2QI
 	     (match_dup 1)
 	     (parallel [(const_int 11)
 			(const_int 15)])))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddbq\t{%1, %0|%0, %1}"
+"vphaddbq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddwd"
+(define_insn "xop_phaddwd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (sign_extend:V4SI
 	  (vec_select:V4HI
 	   (match_operand:V8HI 1 "nonimmediate_operand" "xm")
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)
 		      (const_int 5)
 		      (const_int 7)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddwd\t{%1, %0|%0, %1}"
+"vphaddwd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddwq"
+(define_insn "xop_phaddwq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (plus:V2DI
 	  (sign_extend:V2DI
 	   (vec_select:V2HI
 	  (sign_extend:V2DI
 	   (vec_select:V2HI
 	    (match_dup 1)
 	    (parallel [(const_int 3)
 		       (const_int 7)]))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddwq\t{%1, %0|%0, %1}"
+"vphaddwq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phadddq"
+(define_insn "xop_phadddq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (sign_extend:V2DI
 	  (vec_select:V2SI
 	   (match_operand:V4SI 1 "nonimmediate_operand" "xm")
 	 (sign_extend:V2DI
 	  (vec_select:V2SI
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phadddq\t{%1, %0|%0, %1}"
+"vphadddq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddubw"
+(define_insn "xop_phaddubw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(plus:V8HI
 	 (zero_extend:V8HI
 	  (vec_select:V8QI
 	   (match_operand:V16QI 1 "nonimmediate_operand" "xm")
 		      (const_int 7)
 		      (const_int 9)
 		      (const_int 11)
 		      (const_int 13)
 		      (const_int 15)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddubw\t{%1, %0|%0, %1}"
+"vphaddubw\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddubd"
+(define_insn "xop_phaddubd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (plus:V4SI
 	  (zero_extend:V4SI
 	   (vec_select:V4QI
 	    (match_dup 1)
 	    (parallel [(const_int 3)
 		       (const_int 7)
 		       (const_int 11)
 		       (const_int 15)]))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddubd\t{%1, %0|%0, %1}"
+"vphaddubd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddubq"
+(define_insn "xop_phaddubq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (plus:V2DI
 	  (plus:V2DI
 	   (zero_extend:V2DI
 	   (zero_extend:V2DI
 	    (vec_select:V2QI
 	     (match_dup 1)
 	     (parallel [(const_int 11)
 			(const_int 15)])))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddubq\t{%1, %0|%0, %1}"
+"vphaddubq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phadduwd"
+(define_insn "xop_phadduwd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(plus:V4SI
 	 (zero_extend:V4SI
 	  (vec_select:V4HI
 	   (match_operand:V8HI 1 "nonimmediate_operand" "xm")
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)
 		      (const_int 5)
 		      (const_int 7)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phadduwd\t{%1, %0|%0, %1}"
+"vphadduwd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phadduwq"
+(define_insn "xop_phadduwq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (plus:V2DI
 	  (zero_extend:V2DI
 	   (vec_select:V2HI
 	  (zero_extend:V2DI
 	   (vec_select:V2HI
 	    (match_dup 1)
 	    (parallel [(const_int 3)
 		       (const_int 7)]))))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phadduwq\t{%1, %0|%0, %1}"
+"vphadduwq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phaddudq"
+(define_insn "xop_phaddudq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(plus:V2DI
 	 (zero_extend:V2DI
 	  (vec_select:V2SI
 	   (match_operand:V4SI 1 "nonimmediate_operand" "xm")
 	 (zero_extend:V2DI
 	  (vec_select:V2SI
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phaddudq\t{%1, %0|%0, %1}"
+"vphaddudq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phsubbw"
+(define_insn "xop_phsubbw"
 [(set (match_operand:V8HI 0 "register_operand" "=x")
 	(minus:V8HI
 	 (sign_extend:V8HI
 	  (vec_select:V8QI
 	   (match_operand:V16QI 1 "nonimmediate_operand" "xm")
 		      (const_int 7)
 		      (const_int 9)
 		      (const_int 11)
 		      (const_int 13)
 		      (const_int 15)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phsubbw\t{%1, %0|%0, %1}"
+"vphsubbw\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phsubwd"
+(define_insn "xop_phsubwd"
 [(set (match_operand:V4SI 0 "register_operand" "=x")
 	(minus:V4SI
 	 (sign_extend:V4SI
 	  (vec_select:V4HI
 	   (match_operand:V8HI 1 "nonimmediate_operand" "xm")
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)
 		      (const_int 5)
 		      (const_int 7)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phsubwd\t{%1, %0|%0, %1}"
+"vphsubwd\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-(define_insn "sse5_phsubdq"
+(define_insn "xop_phsubdq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(minus:V2DI
 	 (sign_extend:V2DI
 	  (vec_select:V2SI
 	   (match_operand:V4SI 1 "nonimmediate_operand" "xm")
 	 (sign_extend:V2DI
 	  (vec_select:V2SI
 	   (match_dup 1)
 	   (parallel [(const_int 1)
 		      (const_int 3)])))))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"phsubdq\t{%1, %0|%0, %1}"
+"vphsubdq\t{%1, %0|%0, %1}"
 [(set_attr "type" "sseiadd1")])
-;; SSE5 permute instructions
+;; XOP permute instructions
-(define_insn "sse5_pperm"
+(define_insn "xop_pperm"
-[(set (match_operand:V16QI 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:V16QI 0 "register_operand" "=x,x")
 	(unspec:V16QI
-	  [(match_operand:V16QI 1 "nonimmediate_operand" "0,0,x,xm")
+	  [(match_operand:V16QI 1 "register_operand" "x,x")
-	   (match_operand:V16QI 2 "nonimmediate_operand" "x,xm,xm,x")
+	   (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
-	   (match_operand:V16QI 3 "nonimmediate_operand" "xm,x,0,0")]
+	   (match_operand:V16QI 3 "nonimmediate_operand" "xm,x")]
-	  UNSPEC_SSE5_PERMUTE))]
+	  UNSPEC_XOP_PERMUTE))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
-"pperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sse4arg")
 (set_attr "mode" "TI")])
-;; The following are for the various unpack insns which doesn't need the first
+;; XOP pack instructions that combine two vectors into a smaller vector
-;; source operand, so we can just use the output operand for the first operand.
+(define_insn "xop_pperm_pack_v2di_v4si"
-;; This allows either of the other two operands to be a memory operand.  We
-;; can't just use the first operand as an argument to the normal pperm because
-;; then an output only argument, suddenly becomes an input operand.
-(define_insn "sse5_pperm_zero_v16qi_v8hi"
-[(set (match_operand:V8HI 0 "register_operand" "=x,x")
-	(zero_extend:V8HI
-	 (vec_select:V8QI
-	  (match_operand:V16QI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V16QImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-(define_insn "sse5_pperm_sign_v16qi_v8hi"
-[(set (match_operand:V8HI 0 "register_operand" "=x,x")
-	(sign_extend:V8HI
-	 (vec_select:V8QI
-	  (match_operand:V16QI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V16QImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-(define_insn "sse5_pperm_zero_v8hi_v4si"
 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
-	(zero_extend:V4SI
-	 (vec_select:V4HI
-	  (match_operand:V8HI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V8HImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-(define_insn "sse5_pperm_sign_v8hi_v4si"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x")
-	(sign_extend:V4SI
-	 (vec_select:V4HI
-	  (match_operand:V8HI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V8HImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-(define_insn "sse5_pperm_zero_v4si_v2di"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x")
-	(zero_extend:V2DI
-	 (vec_select:V2SI
-	  (match_operand:V4SI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V4SImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-(define_insn "sse5_pperm_sign_v4si_v2di"
-[(set (match_operand:V2DI 0 "register_operand" "=x,x")
-	(sign_extend:V2DI
-	 (vec_select:V2SI
-	  (match_operand:V4SI 1 "nonimmediate_operand" "xm,x")
-	  (match_operand 2 "" ""))))	;; parallel with const_int's
-(use (match_operand:V16QI 3 "nonimmediate_operand" "x,xm"))]
-"TARGET_SSE5
-&& (register_operand (operands[1], V4SImode)
-|| register_operand (operands[2], V16QImode))"
-"pperm\t{%3, %1, %0, %0|%0, %0, %1, %3}"
-[(set_attr "type" "sseadd")
-(set_attr "mode" "TI")])
-;; SSE5 pack instructions that combine two vectors into a smaller vector
-(define_insn "sse5_pperm_pack_v2di_v4si"
-[(set (match_operand:V4SI 0 "register_operand" "=x,x,x,x")
 	(vec_concat:V4SI
 	 (truncate:V2SI
-	  (match_operand:V2DI 1 "nonimmediate_operand" "0,0,x,xm"))
+	  (match_operand:V2DI 1 "register_operand" "x,x"))
 	 (truncate:V2SI
-	  (match_operand:V2DI 2 "nonimmediate_operand" "x,xm,xm,x"))))
+	  (match_operand:V2DI 2 "nonimmediate_operand" "x,m"))))
-(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x,0,0"))]
+(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
-"pperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sse4arg")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pperm_pack_v4si_v8hi"
+(define_insn "xop_pperm_pack_v4si_v8hi"
-[(set (match_operand:V8HI 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:V8HI 0 "register_operand" "=x,x")
 	(vec_concat:V8HI
 	 (truncate:V4HI
-	  (match_operand:V4SI 1 "nonimmediate_operand" "0,0,x,xm"))
+	  (match_operand:V4SI 1 "register_operand" "x,x"))
 	 (truncate:V4HI
-	  (match_operand:V4SI 2 "nonimmediate_operand" "x,xm,xm,x"))))
+	  (match_operand:V4SI 2 "nonimmediate_operand" "x,m"))))
-(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x,0,0"))]
+(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
-"pperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sse4arg")
 (set_attr "mode" "TI")])
-(define_insn "sse5_pperm_pack_v8hi_v16qi"
+(define_insn "xop_pperm_pack_v8hi_v16qi"
-[(set (match_operand:V16QI 0 "register_operand" "=x,x,x,x")
+[(set (match_operand:V16QI 0 "register_operand" "=x,x")
 	(vec_concat:V16QI
 	 (truncate:V8QI
-	  (match_operand:V8HI 1 "nonimmediate_operand" "0,0,x,xm"))
+	  (match_operand:V8HI 1 "register_operand" "x,x"))
 	 (truncate:V8QI
-	  (match_operand:V8HI 2 "nonimmediate_operand" "x,xm,xm,x"))))
+	  (match_operand:V8HI 2 "nonimmediate_operand" "x,m"))))
-(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x,0,0"))]
+(use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
-"pperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+"vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sse4arg")
 (set_attr "mode" "TI")])
-;; Floating point permutation (permps, permpd)
+;; XOP packed rotate instructions
-(define_insn "sse5_perm<mode>"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x,x,x,x")
-	(unspec:SSEMODEF2P
-	 [(match_operand:SSEMODEF2P 1 "nonimmediate_operand" "0,0,x,xm")
-	  (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "x,xm,xm,x")
-	  (match_operand:V16QI 3 "nonimmediate_operand" "xm,x,0,0")]
-	 UNSPEC_SSE5_PERMUTE))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1, false)"
-"perm<ssemodesuffixf4>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
-[(set_attr "type" "sse4arg")
-(set_attr "mode" "<MODE>")])
-;; SSE5 packed rotate instructions
 (define_expand "rotl<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "")
 	(rotate:SSEMODE1248
 	 (match_operand:SSEMODE1248 1 "nonimmediate_operand" "")
 	 (match_operand:SI 2 "general_operand")))]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 /* If we were given a scalar, convert it to parallel */
 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
 {
 rtvec vs = rtvec_alloc (<ssescalarnum>);
 for (i = 0; i < <ssescalarnum>; i++)
 	RTVEC_ELT (vs, i) = op2;
 emit_insn (gen_vec_init<mode> (reg, par));
-emit_insn (gen_sse5_vrotl<mode>3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
 DONE;
 }
 })
 (define_expand "rotr<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "")
 	(rotatert:SSEMODE1248
 	 (match_operand:SSEMODE1248 1 "nonimmediate_operand" "")
 	 (match_operand:SI 2 "general_operand")))]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 /* If we were given a scalar, convert it to parallel */
 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
 {
 rtvec vs = rtvec_alloc (<ssescalarnum>);
 for (i = 0; i < <ssescalarnum>; i++)
 	RTVEC_ELT (vs, i) = op2;
 emit_insn (gen_vec_init<mode> (reg, par));
 emit_insn (gen_neg<mode>2 (neg, reg));
-emit_insn (gen_sse5_vrotl<mode>3 (operands[0], operands[1], neg));
+emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], neg));
 DONE;
 }
 })
-(define_insn "sse5_rotl<mode>3"
+(define_insn "xop_rotl<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(rotate:SSEMODE1248
 	 (match_operand:SSEMODE1248 1 "nonimmediate_operand" "xm")
 	 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"prot<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
+"vprot<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
-(set_attr "mode" "TI")])
+(set_attr "length_immediate" "1")
+(set_attr "mode" "TI")])
-(define_insn "sse5_rotr<mode>3"
+(define_insn "xop_rotr<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(rotatert:SSEMODE1248
 	 (match_operand:SSEMODE1248 1 "nonimmediate_operand" "xm")
 	 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 operands[3] = GEN_INT ((<ssescalarnum> * 8) - INTVAL (operands[2]));
-return \"prot<ssevecsize>\t{%3, %1, %0|%0, %1, %3}\";
+return \"vprot<ssevecsize>\t{%3, %1, %0|%0, %1, %3}\";
 }
 [(set_attr "type" "sseishft")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_expand "vrotr<mode>3"
 [(match_operand:SSEMODE1248 0 "register_operand" "")
 (match_operand:SSEMODE1248 1 "register_operand" "")
 (match_operand:SSEMODE1248 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtx reg = gen_reg_rtx (<MODE>mode);
 emit_insn (gen_neg<mode>2 (reg, operands[2]));
-emit_insn (gen_sse5_vrotl<mode>3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
 DONE;
 })
 (define_expand "vrotl<mode>3"
 [(match_operand:SSEMODE1248 0 "register_operand" "")
 (match_operand:SSEMODE1248 1 "register_operand" "")
 (match_operand:SSEMODE1248 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
-emit_insn (gen_sse5_vrotl<mode>3 (operands[0], operands[1], operands[2]));
+emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], operands[2]));
 DONE;
 })
-(define_insn "sse5_vrotl<mode>3"
+(define_insn "xop_vrotl<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x,x")
 	(if_then_else:SSEMODE1248
 	 (ge:SSEMODE1248
-	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm,x")
+	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "x,m")
 	  (const_int 0))
 	 (rotate:SSEMODE1248
-	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "x,xm")
+	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "xm,x")
 	  (match_dup 2))
 	 (rotatert:SSEMODE1248
 	  (match_dup 1)
 	  (neg:SSEMODE1248 (match_dup 2)))))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 3, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-"prot<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
+"vprot<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
-(set_attr "mode" "TI")])
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_extra" "2")
-;; SSE5 packed shift instructions.
+(set_attr "mode" "TI")])
+;; XOP packed shift instructions.
 ;; FIXME: add V2DI back in
 (define_expand "vlshr<mode>3"
 [(match_operand:SSEMODE124 0 "register_operand" "")
 (match_operand:SSEMODE124 1 "register_operand" "")
 (match_operand:SSEMODE124 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtx neg = gen_reg_rtx (<MODE>mode);
 emit_insn (gen_neg<mode>2 (neg, operands[2]));
-emit_insn (gen_sse5_lshl<mode>3 (operands[0], operands[1], neg));
+emit_insn (gen_xop_lshl<mode>3 (operands[0], operands[1], neg));
 DONE;
 })
 (define_expand "vashr<mode>3"
 [(match_operand:SSEMODE124 0 "register_operand" "")
 (match_operand:SSEMODE124 1 "register_operand" "")
 (match_operand:SSEMODE124 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtx neg = gen_reg_rtx (<MODE>mode);
 emit_insn (gen_neg<mode>2 (neg, operands[2]));
-emit_insn (gen_sse5_ashl<mode>3 (operands[0], operands[1], neg));
+emit_insn (gen_xop_ashl<mode>3 (operands[0], operands[1], neg));
 DONE;
 })
 (define_expand "vashl<mode>3"
 [(match_operand:SSEMODE124 0 "register_operand" "")
 (match_operand:SSEMODE124 1 "register_operand" "")
 (match_operand:SSEMODE124 2 "register_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
-emit_insn (gen_sse5_ashl<mode>3 (operands[0], operands[1], operands[2]));
+emit_insn (gen_xop_ashl<mode>3 (operands[0], operands[1], operands[2]));
 DONE;
 })
-(define_insn "sse5_ashl<mode>3"
+(define_insn "xop_ashl<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x,x")
 	(if_then_else:SSEMODE1248
 	 (ge:SSEMODE1248
-	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm,x")
+	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "x,m")
 	  (const_int 0))
 	 (ashift:SSEMODE1248
-	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "x,xm")
+	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "xm,x")
 	  (match_dup 2))
 	 (ashiftrt:SSEMODE1248
 	  (match_dup 1)
 	  (neg:SSEMODE1248 (match_dup 2)))))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 3, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-"psha<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
+"vpsha<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
-(set_attr "mode" "TI")])
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_extra" "2")
-(define_insn "sse5_lshl<mode>3"
+(set_attr "mode" "TI")])
+(define_insn "xop_lshl<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x,x")
 	(if_then_else:SSEMODE1248
 	 (ge:SSEMODE1248
-	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm,x")
+	  (match_operand:SSEMODE1248 2 "nonimmediate_operand" "x,m")
 	  (const_int 0))
 	 (ashift:SSEMODE1248
-	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "x,xm")
+	  (match_operand:SSEMODE1248 1 "nonimmediate_operand" "xm,x")
 	  (match_dup 2))
 	 (lshiftrt:SSEMODE1248
 	  (match_dup 1)
 	  (neg:SSEMODE1248 (match_dup 2)))))]
-"TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 3, true, 1, false)"
+"TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-"pshl<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
+"vpshl<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sseishft")
-(set_attr "mode" "TI")])
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_extra" "2")
-;; SSE2 doesn't have some shift varients, so define versions for SSE5
+(set_attr "mode" "TI")])
+;; SSE2 doesn't have some shift varients, so define versions for XOP
 (define_expand "ashlv16qi3"
 [(match_operand:V16QI 0 "register_operand" "")
 (match_operand:V16QI 1 "register_operand" "")
 (match_operand:SI 2 "nonmemory_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtvec vs = rtvec_alloc (16);
 rtx par = gen_rtx_PARALLEL (V16QImode, vs);
 rtx reg = gen_reg_rtx (V16QImode);
 int i;
 for (i = 0; i < 16; i++)
 RTVEC_ELT (vs, i) = operands[2];
 emit_insn (gen_vec_initv16qi (reg, par));
-emit_insn (gen_sse5_ashlv16qi3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], reg));
 DONE;
 })
 (define_expand "lshlv16qi3"
 [(match_operand:V16QI 0 "register_operand" "")
 (match_operand:V16QI 1 "register_operand" "")
 (match_operand:SI 2 "nonmemory_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtvec vs = rtvec_alloc (16);
 rtx par = gen_rtx_PARALLEL (V16QImode, vs);
 rtx reg = gen_reg_rtx (V16QImode);
 int i;
 for (i = 0; i < 16; i++)
 RTVEC_ELT (vs, i) = operands[2];
 emit_insn (gen_vec_initv16qi (reg, par));
-emit_insn (gen_sse5_lshlv16qi3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_lshlv16qi3 (operands[0], operands[1], reg));
 DONE;
 })
 (define_expand "ashrv16qi3"
 [(match_operand:V16QI 0 "register_operand" "")
 (match_operand:V16QI 1 "register_operand" "")
 (match_operand:SI 2 "nonmemory_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtvec vs = rtvec_alloc (16);
 rtx par = gen_rtx_PARALLEL (V16QImode, vs);
 rtx reg = gen_reg_rtx (V16QImode);
 int i;
-rtx ele = ((GET_CODE (operands[2]) == CONST_INT)
+rtx ele = ((CONST_INT_P (operands[2]))
 	     ? GEN_INT (- INTVAL (operands[2]))
 	     : operands[2]);
 for (i = 0; i < 16; i++)
 RTVEC_ELT (vs, i) = ele;
 emit_insn (gen_vec_initv16qi (reg, par));
-if (GET_CODE (operands[2]) != CONST_INT)
+if (!CONST_INT_P (operands[2]))
 {
 rtx neg = gen_reg_rtx (V16QImode);
 emit_insn (gen_negv16qi2 (neg, reg));
-emit_insn (gen_sse5_ashlv16qi3 (operands[0], operands[1], neg));
+emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], neg));
 }
 else
-emit_insn (gen_sse5_ashlv16qi3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], reg));
 DONE;
 })
 (define_expand "ashrv2di3"
 [(match_operand:V2DI 0 "register_operand" "")
 (match_operand:V2DI 1 "register_operand" "")
 (match_operand:DI 2 "nonmemory_operand" "")]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 rtvec vs = rtvec_alloc (2);
 rtx par = gen_rtx_PARALLEL (V2DImode, vs);
 rtx reg = gen_reg_rtx (V2DImode);
 rtx ele;
-if (GET_CODE (operands[2]) == CONST_INT)
+if (CONST_INT_P (operands[2]))
 ele = GEN_INT (- INTVAL (operands[2]));
 else if (GET_MODE (operands[2]) != DImode)
 {
 rtx move = gen_reg_rtx (DImode);
 ele = gen_reg_rtx (DImode);
 }
 RTVEC_ELT (vs, 0) = ele;
 RTVEC_ELT (vs, 1) = ele;
 emit_insn (gen_vec_initv2di (reg, par));
-emit_insn (gen_sse5_ashlv2di3 (operands[0], operands[1], reg));
+emit_insn (gen_xop_ashlv2di3 (operands[0], operands[1], reg));
 DONE;
 })
-;; SSE5 FRCZ support
+;; XOP FRCZ support
 ;; parallel insns
-(define_insn "sse5_frcz<mode>2"
+(define_insn "xop_frcz<mode>2"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	(unspec:SSEMODEF2P
 	 [(match_operand:SSEMODEF2P 1 "nonimmediate_operand" "xm")]
 	 UNSPEC_FRCZ))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"frcz<ssemodesuffixf4>\t{%1, %0|%0, %1}"
+"vfrcz<ssemodesuffixf4>\t{%1, %0|%0, %1}"
 [(set_attr "type" "ssecvt1")
-(set_attr "prefix_extra" "1")
 (set_attr "mode" "<MODE>")])
 ;; scalar insns
-(define_insn "sse5_vmfrcz<mode>2"
+(define_insn "xop_vmfrcz<mode>2"
 [(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
 	(vec_merge:SSEMODEF2P
 	  (unspec:SSEMODEF2P
 	   [(match_operand:SSEMODEF2P 2 "nonimmediate_operand" "xm")]
 	   UNSPEC_FRCZ)
 	  (match_operand:SSEMODEF2P 1 "register_operand" "0")
 	  (const_int 1)))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"frcz<ssemodesuffixf2s>\t{%2, %0|%0, %2}"
+"vfrcz<ssemodesuffixf2s>\t{%2, %0|%0, %2}"
 [(set_attr "type" "ssecvt1")
-(set_attr "prefix_extra" "1")
 (set_attr "mode" "<MODE>")])
-(define_insn "sse5_cvtph2ps"
+(define_insn "xop_frcz<mode>2256"
-[(set (match_operand:V4SF 0 "register_operand" "=x")
+[(set (match_operand:FMA4MODEF4 0 "register_operand" "=x")
-	(unspec:V4SF [(match_operand:V4HI 1 "nonimmediate_operand" "xm")]
+	(unspec:FMA4MODEF4
-		     UNSPEC_CVTPH2PS))]
+	 [(match_operand:FMA4MODEF4 1 "nonimmediate_operand" "xm")]
-"TARGET_SSE5"
+	 UNSPEC_FRCZ))]
-"cvtph2ps\t{%1, %0|%0, %1}"
+"TARGET_XOP"
-[(set_attr "type" "ssecvt")
+"vfrcz<fma4modesuffixf4>\t{%1, %0|%0, %1}"
-(set_attr "mode" "V4SF")])
+[(set_attr "type" "ssecvt1")
-(define_insn "sse5_cvtps2ph"
-[(set (match_operand:V4HI 0 "nonimmediate_operand" "=xm")
-	(unspec:V4HI [(match_operand:V4SF 1 "register_operand" "x")]
-		     UNSPEC_CVTPS2PH))]
-"TARGET_SSE5"
-"cvtps2ph\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssecvt")
-(set_attr "mode" "V4SF")])
-;; Scalar versions of the com instructions that use vector types that are
-;; called from the intrinsics.  Unlike the the other s{s,d} instructions, the
-;; com instructions fill in 0's in the upper bits instead of leaving them
-;; unmodified, so we use const_vector of 0 instead of match_dup.
-(define_expand "sse5_vmmaskcmp<mode>3"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "")
-	(vec_merge:SSEMODEF2P
-	 (match_operator:SSEMODEF2P 1 "sse5_comparison_float_operator"
-	  [(match_operand:SSEMODEF2P 2 "register_operand" "")
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "")])
-	 (match_dup 4)
-	 (const_int 1)))]
-"TARGET_SSE5"
-{
-operands[4] = CONST0_RTX (<MODE>mode);
-})
-(define_insn "*sse5_vmmaskcmp<mode>3"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
-	(vec_merge:SSEMODEF2P
-	 (match_operator:SSEMODEF2P 1 "sse5_comparison_float_operator"
-	  [(match_operand:SSEMODEF2P 2 "register_operand" "x")
-	   (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm")])
-	  (match_operand:SSEMODEF2P 4 "")
-	  (const_int 1)))]
-"TARGET_SSE5"
-"com%Y1<ssemodesuffixf2s>\t{%3, %2, %0|%0, %2, %3}"
-[(set_attr "type" "sse4arg")
-(set_attr "mode" "<ssescalarmode>")])
-;; We don't have a comparison operator that always returns true/false, so
-;; handle comfalse and comtrue specially.
-(define_insn "sse5_com_tf<mode>3"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
-	(unspec:SSEMODEF2P
-	 [(match_operand:SSEMODEF2P 1 "register_operand" "x")
-	  (match_operand:SSEMODEF2P 2 "nonimmediate_operand" "xm")
-	  (match_operand:SI 3 "const_int_operand" "n")]
-	 UNSPEC_SSE5_TRUEFALSE))]
-"TARGET_SSE5"
-{
-const char *ret = NULL;
-switch (INTVAL (operands[3]))
-{
-case COM_FALSE_S:
-ret = \"comfalses<ssemodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}\";
-break;
-case COM_FALSE_P:
-ret = \"comfalsep<ssemodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}\";
-break;
-case COM_TRUE_S:
-ret = \"comfalses<ssemodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}\";
-break;
-case COM_TRUE_P:
-ret = \"comfalsep<ssemodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}\";
-break;
-default:
-gcc_unreachable ();
-}
-return ret;
-}
-[(set_attr "type" "ssecmp")
 (set_attr "mode" "<MODE>")])
-(define_insn "sse5_maskcmp<mode>3"
+(define_insn "xop_maskcmp<mode>3"
-[(set (match_operand:SSEMODEF2P 0 "register_operand" "=x")
-	(match_operator:SSEMODEF2P 1 "sse5_comparison_float_operator"
-	 [(match_operand:SSEMODEF2P 2 "register_operand" "x")
-	  (match_operand:SSEMODEF2P 3 "nonimmediate_operand" "xm")]))]
-"TARGET_SSE5"
-"com%Y1<ssemodesuffixf4>\t{%3, %2, %0|%0, %2, %3}"
-[(set_attr "type" "ssecmp")
-(set_attr "mode" "<MODE>")])
-(define_insn "sse5_maskcmp<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(match_operator:SSEMODE1248 1 "ix86_comparison_int_operator"
 	 [(match_operand:SSEMODE1248 2 "register_operand" "x")
 	  (match_operand:SSEMODE1248 3 "nonimmediate_operand" "xm")]))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"pcom%Y1<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
+"vpcom%Y1<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "sse4arg")
-(set_attr "mode" "TI")])
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_rep" "0")
-(define_insn "sse5_maskcmp_uns<mode>3"
+(set_attr "prefix_extra" "2")
+(set_attr "length_immediate" "1")
+(set_attr "mode" "TI")])
+(define_insn "xop_maskcmp_uns<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(match_operator:SSEMODE1248 1 "ix86_comparison_uns_operator"
 	 [(match_operand:SSEMODE1248 2 "register_operand" "x")
 	  (match_operand:SSEMODE1248 3 "nonimmediate_operand" "xm")]))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"pcom%Y1u<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
+"vpcom%Y1u<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssecmp")
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_rep" "0")
+(set_attr "prefix_extra" "2")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 ;; Version of pcom*u* that is called from the intrinsics that allows pcomequ*
 ;; and pcomneu* not to be converted to the signed ones in case somebody needs
 ;; the exact instruction generated for the intrinsic.
-(define_insn "sse5_maskcmp_uns2<mode>3"
+(define_insn "xop_maskcmp_uns2<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(unspec:SSEMODE1248
 	 [(match_operator:SSEMODE1248 1 "ix86_comparison_uns_operator"
 	  [(match_operand:SSEMODE1248 2 "register_operand" "x")
 	   (match_operand:SSEMODE1248 3 "nonimmediate_operand" "xm")])]
-	 UNSPEC_SSE5_UNSIGNED_CMP))]
+	 UNSPEC_XOP_UNSIGNED_CMP))]
-"TARGET_SSE5"
+"TARGET_XOP"
-"pcom%Y1u<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
+"vpcom%Y1u<ssevecsize>\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "ssecmp")
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_extra" "2")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 ;; Pcomtrue and pcomfalse support.  These are useless instructions, but are
 ;; being added here to be complete.
-(define_insn "sse5_pcom_tf<mode>3"
+(define_insn "xop_pcom_tf<mode>3"
 [(set (match_operand:SSEMODE1248 0 "register_operand" "=x")
 	(unspec:SSEMODE1248
 	  [(match_operand:SSEMODE1248 1 "register_operand" "x")
 	   (match_operand:SSEMODE1248 2 "nonimmediate_operand" "xm")
 	   (match_operand:SI 3 "const_int_operand" "n")]
-	  UNSPEC_SSE5_TRUEFALSE))]
+	  UNSPEC_XOP_TRUEFALSE))]
-"TARGET_SSE5"
+"TARGET_XOP"
 {
 return ((INTVAL (operands[3]) != 0)
-	  ? "pcomtrue<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
+	  ? "vpcomtrue<ssevecsize>\t{%2, %1, %0|%0, %1, %2}"
-	  : "pcomfalse<ssevecsize>\t{%2, %1, %0|%0, %1, %2}");
+	  : "vpcomfalse<ssevecsize>\t{%2, %1, %0|%0, %1, %2}");
 }
 [(set_attr "type" "ssecmp")
-(set_attr "mode" "TI")])
+(set_attr "prefix_data16" "0")
+(set_attr "prefix_extra" "2")
+(set_attr "length_immediate" "1")
+(set_attr "mode" "TI")])
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 (define_insn "*avx_aesenc"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 	(unspec:V2DI [(match_operand:V2DI 1 "register_operand" "x")
 		       (match_operand:V2DI 2 "nonimmediate_operand" "xm")]
 		      UNSPEC_AESENC))]
 "TARGET_AES && TARGET_AVX"
 "vaesenc\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "aesenc"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		       (match_operand:V2DI 2 "nonimmediate_operand" "xm")]
 		      UNSPEC_AESENCLAST))]
 "TARGET_AES && TARGET_AVX"
 "vaesenclast\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "aesenclast"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		       (match_operand:V2DI 2 "nonimmediate_operand" "xm")]
 		      UNSPEC_AESDEC))]
 "TARGET_AES && TARGET_AVX"
 "vaesdec\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "aesdec"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		       (match_operand:V2DI 2 "nonimmediate_operand" "xm")]
 		      UNSPEC_AESDECLAST))]
 "TARGET_AES && TARGET_AVX"
 "vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "aesdeclast"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		     UNSPEC_AESKEYGENASSIST))]
 "TARGET_AES"
 "%vaeskeygenassist\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "maybe_vex")
 (set_attr "mode" "TI")])
 (define_insn "*vpclmulqdq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		      (match_operand:SI 3 "const_0_to_255_operand" "n")]
 		     UNSPEC_PCLMUL))]
 "TARGET_PCLMUL && TARGET_AVX"
 "vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "TI")])
 (define_insn "pclmulqdq"
 [(set (match_operand:V2DI 0 "register_operand" "=x")
 		     UNSPEC_PCLMUL))]
 "TARGET_PCLMUL"
 "pclmulqdq\t{%3, %2, %0|%0, %2, %3}"
 [(set_attr "type" "sselog1")
 (set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "mode" "TI")])
 (define_expand "avx_vzeroall"
 [(match_par_dup 0 [(const_int 0)])]
 "TARGET_AVX"
 		     CONST0_RTX (V8SImode));
 })
 (define_insn "*avx_vzeroall"
 [(match_parallel 0 "vzeroall_operation"
-[(unspec_volatile [(const_int 0)] UNSPECV_VZEROALL)
+[(unspec_volatile [(const_int 0)] UNSPECV_VZEROALL)])]
-(set (match_operand 1 "register_operand" "=x")
-(match_operand 2 "const0_operand" "X"))])]
 "TARGET_AVX"
 "vzeroall"
 [(set_attr "type" "sse")
+(set_attr "modrm" "0")
 (set_attr "memory" "none")
 (set_attr "prefix" "vex")
 (set_attr "mode" "OI")])
 ;; vzeroupper clobbers the upper 128bits of AVX registers.
-(define_insn "avx_vzeroupper"
+(define_expand "avx_vzeroupper"
-[(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)
+[(match_par_dup 0 [(const_int 0)])]
-(clobber (reg:V8SI XMM0_REG))
+"TARGET_AVX"
-(clobber (reg:V8SI XMM1_REG))
+{
-(clobber (reg:V8SI XMM2_REG))
+int nregs = TARGET_64BIT ? 16 : 8;
-(clobber (reg:V8SI XMM3_REG))
+int regno;
-(clobber (reg:V8SI XMM4_REG))
-(clobber (reg:V8SI XMM5_REG))
+operands[0] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (nregs + 1));
-(clobber (reg:V8SI XMM6_REG))
-(clobber (reg:V8SI XMM7_REG))]
+XVECEXP (operands[0], 0, 0)
-"TARGET_AVX && !TARGET_64BIT"
+= gen_rtx_UNSPEC_VOLATILE (VOIDmode, gen_rtvec (1, const0_rtx),
+			       UNSPECV_VZEROUPPER);
+for (regno = 0; regno < nregs; regno++)
+XVECEXP (operands[0], 0, regno + 1)
+= gen_rtx_CLOBBER (VOIDmode,
+			 gen_rtx_REG (V8SImode, SSE_REGNO (regno)));
+})
+(define_insn "*avx_vzeroupper"
+[(match_parallel 0 "vzeroupper_operation"
+[(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)])]
+"TARGET_AVX"
 "vzeroupper"
 [(set_attr "type" "sse")
+(set_attr "modrm" "0")
 (set_attr "memory" "none")
 (set_attr "prefix" "vex")
 (set_attr "mode" "OI")])
-(define_insn "avx_vzeroupper_rex64"
+(define_insn_and_split "vec_dup<mode>"
-[(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)
+[(set (match_operand:AVX256MODE24P 0 "register_operand" "=x,x")
-(clobber (reg:V8SI XMM0_REG))
+	(vec_duplicate:AVX256MODE24P
-(clobber (reg:V8SI XMM1_REG))
+	  (match_operand:<avxscalarmode> 1 "nonimmediate_operand" "m,?x")))]
-(clobber (reg:V8SI XMM2_REG))
+"TARGET_AVX"
-(clobber (reg:V8SI XMM3_REG))
+"@
-(clobber (reg:V8SI XMM4_REG))
+vbroadcasts<avxmodesuffixf2c>\t{%1, %0|%0, %1}
-(clobber (reg:V8SI XMM5_REG))
+#"
-(clobber (reg:V8SI XMM6_REG))
+"&& reload_completed && REG_P (operands[1])"
-(clobber (reg:V8SI XMM7_REG))
+[(set (match_dup 2) (vec_duplicate:<avxhalfvecmode> (match_dup 1)))
-(clobber (reg:V8SI XMM8_REG))
+(set (match_dup 0) (vec_concat:AVX256MODE24P (match_dup 2) (match_dup 2)))]
-(clobber (reg:V8SI XMM9_REG))
+{
-(clobber (reg:V8SI XMM10_REG))
+operands[2] = gen_rtx_REG (<avxhalfvecmode>mode, REGNO (operands[0]));
-(clobber (reg:V8SI XMM11_REG))
+}
-(clobber (reg:V8SI XMM12_REG))
+[(set_attr "type" "ssemov")
-(clobber (reg:V8SI XMM13_REG))
+(set_attr "prefix_extra" "1")
-(clobber (reg:V8SI XMM14_REG))
+(set_attr "prefix" "vex")
-(clobber (reg:V8SI XMM15_REG))]
+(set_attr "mode" "V8SF")])
-"TARGET_AVX && TARGET_64BIT"
-"vzeroupper"
+(define_insn "avx_vbroadcastf128_<mode>"
-[(set_attr "type" "sse")
+[(set (match_operand:AVX256MODE 0 "register_operand" "=x,x,x")
-(set_attr "memory" "none")
+	(vec_concat:AVX256MODE
-(set_attr "prefix" "vex")
+	  (match_operand:<avxhalfvecmode> 1 "nonimmediate_operand" "m,0,?x")
-(set_attr "mode" "OI")])
+	  (match_dup 1)))]
+"TARGET_AVX"
-(define_insn "avx_vpermil<mode>"
+"@
+vbroadcastf128\t{%1, %0|%0, %1}
+vinsertf128\t{$1, %1, %0, %0|%0, %0, %1, 1}
+vperm2f128\t{$0, %t1, %t1, %0|%0, %t1, %t1, 0}"
+[(set_attr "type" "ssemov,sselog1,sselog1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "0,1,1")
+(set_attr "prefix" "vex")
+(set_attr "mode" "V4SF,V8SF,V8SF")])
+;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
+;; If it so happens that the input is in memory, use vbroadcast.
+;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
+(define_insn "*avx_vperm_broadcast_v4sf"
+[(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
+	(vec_select:V4SF
+	  (match_operand:V4SF 1 "nonimmediate_operand" "m,o,x")
+	  (match_parallel 2 "avx_vbroadcast_operand"
+	    [(match_operand 3 "const_int_operand" "C,n,n")])))]
+"TARGET_AVX"
+{
+int elt = INTVAL (operands[3]);
+switch (which_alternative)
+{
+case 0:
+case 1:
+operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
+return "vbroadcastss\t{%1, %0|%0, %1}";
+case 2:
+operands[2] = GEN_INT (elt * 0x55);
+return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
+default:
+gcc_unreachable ();
+}
+}
+[(set_attr "type" "ssemov,ssemov,sselog1")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "0,0,1")
+(set_attr "prefix" "vex")
+(set_attr "mode" "SF,SF,V4SF")])
+(define_insn_and_split "*avx_vperm_broadcast_<mode>"
+[(set (match_operand:AVX256MODEF2P 0 "register_operand" "=x,x,x")
+	(vec_select:AVX256MODEF2P
+	  (match_operand:AVX256MODEF2P 1 "nonimmediate_operand" "m,o,?x")
+	  (match_parallel 2 "avx_vbroadcast_operand"
+	    [(match_operand 3 "const_int_operand" "C,n,n")])))]
+"TARGET_AVX"
+"#"
+"&& reload_completed"
+[(set (match_dup 0) (vec_duplicate:AVX256MODEF2P (match_dup 1)))]
+{
+rtx op0 = operands[0], op1 = operands[1];
+int elt = INTVAL (operands[3]);
+if (REG_P (op1))
+{
+int mask;
+/* Shuffle element we care about into all elements of the 128-bit lane.
+	 The other lane gets shuffled too, but we don't care.  */
+if (<MODE>mode == V4DFmode)
+	mask = (elt & 1 ? 15 : 0);
+else
+	mask = (elt & 3) * 0x55;
+emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
+/* Shuffle the lane we care about into both lanes of the dest.  */
+mask = (elt / (<ssescalarnum> / 2)) * 0x11;
+emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
+DONE;
+}
+operands[1] = adjust_address_nv (op1, <avxscalarmode>mode,
+	      			   elt * GET_MODE_SIZE (<avxscalarmode>mode));
+})
+(define_expand "avx_vpermil<mode>"
+[(set (match_operand:AVXMODEFDP 0 "register_operand" "")
+	(vec_select:AVXMODEFDP
+	  (match_operand:AVXMODEFDP 1 "nonimmediate_operand" "")
+	  (match_operand:SI 2 "const_0_to_255_operand" "")))]
+"TARGET_AVX"
+{
+int mask = INTVAL (operands[2]);
+rtx perm[<ssescalarnum>];
+perm[0] = GEN_INT (mask & 1);
+perm[1] = GEN_INT ((mask >> 1) & 1);
+if (<MODE>mode == V4DFmode)
+{
+perm[2] = GEN_INT (((mask >> 2) & 1) + 2);
+perm[3] = GEN_INT (((mask >> 3) & 1) + 2);
+}
+operands[2]
+= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
+})
+(define_expand "avx_vpermil<mode>"
+[(set (match_operand:AVXMODEFSP 0 "register_operand" "")
+	(vec_select:AVXMODEFSP
+	  (match_operand:AVXMODEFSP 1 "nonimmediate_operand" "")
+	  (match_operand:SI 2 "const_0_to_255_operand" "")))]
+"TARGET_AVX"
+{
+int mask = INTVAL (operands[2]);
+rtx perm[<ssescalarnum>];
+perm[0] = GEN_INT (mask & 3);
+perm[1] = GEN_INT ((mask >> 2) & 3);
+perm[2] = GEN_INT ((mask >> 4) & 3);
+perm[3] = GEN_INT ((mask >> 6) & 3);
+if (<MODE>mode == V8SFmode)
+{
+perm[4] = GEN_INT ((mask & 3) + 4);
+perm[5] = GEN_INT (((mask >> 2) & 3) + 4);
+perm[6] = GEN_INT (((mask >> 4) & 3) + 4);
+perm[7] = GEN_INT (((mask >> 6) & 3) + 4);
+}
+operands[2]
+= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
+})
+(define_insn "*avx_vpermilp<mode>"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
-	(unspec:AVXMODEF2P
+	(vec_select:AVXMODEF2P
-	  [(match_operand:AVXMODEF2P 1 "register_operand" "xm")
+	  (match_operand:AVXMODEF2P 1 "nonimmediate_operand" "xm")
-	   (match_operand:SI 2 "const_0_to_<vpermilbits>_operand" "n")]
+	  (match_parallel 2 "avx_vpermilp_<mode>_operand"
-	  UNSPEC_VPERMIL))]
+	    [(match_operand 3 "const_int_operand" "")])))]
 "TARGET_AVX"
-"vpermilp<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
+{
+int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
+operands[2] = GEN_INT (mask);
+return "vpermilp<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}";
+}
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_vpermilvar<mode>3"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
 	   (match_operand:<avxpermvecmode> 2 "nonimmediate_operand" "xm")]
 	  UNSPEC_VPERMIL))]
 "TARGET_AVX"
 "vpermilp<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
-(define_insn "avx_vperm2f128<mode>3"
+(define_expand "avx_vperm2f128<mode>3"
+[(set (match_operand:AVX256MODE2P 0 "register_operand" "")
+	(unspec:AVX256MODE2P
+	  [(match_operand:AVX256MODE2P 1 "register_operand" "")
+	   (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "")
+	   (match_operand:SI 3 "const_0_to_255_operand" "")]
+	  UNSPEC_VPERMIL2F128))]
+"TARGET_AVX"
+{
+int mask = INTVAL (operands[2]);
+if ((mask & 0x88) == 0)
+{
+rtx perm[<ssescalarnum>], t1, t2;
+int i, base, nelt = <ssescalarnum>, nelt2 = nelt / 2;
+base = (mask & 3) * nelt2;
+for (i = 0; i < nelt2; ++i)
+	perm[i] = GEN_INT (base + i);
+base = ((mask >> 4) & 3) * nelt2;
+for (i = 0; i < nelt2; ++i)
+	perm[i + nelt2] = GEN_INT (base + i);
+t2 = gen_rtx_VEC_CONCAT (<ssedoublesizemode>mode,
+			       operands[1], operands[2]);
+t1 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, perm));
+t2 = gen_rtx_VEC_SELECT (<MODE>mode, t2, t1);
+t2 = gen_rtx_SET (VOIDmode, operands[0], t2);
+emit_insn (t2);
+DONE;
+}
+})
+;; Note that bits 7 and 3 of the imm8 allow lanes to be zeroed, which
+;; means that in order to represent this properly in rtl we'd have to
+;; nest *another* vec_concat with a zero operand and do the select from
+;; a 4x wide vector.  That doesn't seem very nice.
+(define_insn "*avx_vperm2f128<mode>_full"
 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
 	(unspec:AVX256MODE2P
 	  [(match_operand:AVX256MODE2P 1 "register_operand" "x")
 	   (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm")
 	   (match_operand:SI 3 "const_0_to_255_operand" "n")]
 	  UNSPEC_VPERMIL2F128))]
 "TARGET_AVX"
 "vperm2f128\t{%3, %2, %1, %0|%0, %1, %2, %3}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
-(define_insn "avx_vbroadcasts<avxmodesuffixf2c><avxmodesuffix>"
+(define_insn "*avx_vperm2f128<mode>_nozero"
-[(set (match_operand:AVXMODEF4P 0 "register_operand" "=x")
+[(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
-	(vec_concat:AVXMODEF4P
+	(vec_select:AVX256MODE2P
-	  (vec_concat:<avxhalfvecmode>
+	  (vec_concat:<ssedoublesizemode>
-	    (match_operand:<avxscalarmode> 1 "memory_operand" "m")
+	    (match_operand:AVX256MODE2P 1 "register_operand" "x")
-	    (match_dup 1))
+	    (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm"))
-	  (vec_concat:<avxhalfvecmode>
+	  (match_parallel 3 "avx_vperm2f128_<mode>_operand"
-	    (match_dup 1)
+	    [(match_operand 4 "const_int_operand" "")])))]
-	    (match_dup 1))))]
+"TARGET_AVX"
-"TARGET_AVX"
+{
-"vbroadcasts<avxmodesuffixf2c>\t{%1, %0|%0, %1}"
+int mask = avx_vperm2f128_parallel (operands[3], <MODE>mode) - 1;
-[(set_attr "type" "ssemov")
+operands[3] = GEN_INT (mask);
-(set_attr "prefix" "vex")
+return "vperm2f128\t{%3, %2, %1, %0|%0, %1, %2, %3}";
-(set_attr "mode" "<avxscalarmode>")])
+}
+[(set_attr "type" "sselog")
-(define_insn "avx_vbroadcastss256"
+(set_attr "prefix_extra" "1")
-[(set (match_operand:V8SF 0 "register_operand" "=x")
+(set_attr "length_immediate" "1")
-	(vec_concat:V8SF
+(set_attr "prefix" "vex")
-	  (vec_concat:V4SF
+(set_attr "mode" "V8SF")])
-	    (vec_concat:V2SF
-	      (match_operand:SF 1 "memory_operand" "m")
-	      (match_dup 1))
-	    (vec_concat:V2SF
-	      (match_dup 1)
-	      (match_dup 1)))
-	  (vec_concat:V4SF
-	    (vec_concat:V2SF
-	      (match_dup 1)
-	      (match_dup 1))
-	    (vec_concat:V2SF
-	      (match_dup 1)
-	      (match_dup 1)))))]
-"TARGET_AVX"
-"vbroadcastss\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssemov")
-(set_attr "prefix" "vex")
-(set_attr "mode" "SF")])
-(define_insn "avx_vbroadcastf128_p<avxmodesuffixf2c>256"
-[(set (match_operand:AVX256MODEF2P 0 "register_operand" "=x")
-	(vec_concat:AVX256MODEF2P
-	  (match_operand:<avxhalfvecmode> 1 "memory_operand" "m")
-	  (match_dup 1)))]
-"TARGET_AVX"
-"vbroadcastf128\t{%1, %0|%0, %1}"
-[(set_attr "type" "ssemov")
-(set_attr "prefix" "vex")
-(set_attr "mode" "V4SF")])
 (define_expand "avx_vinsertf128<mode>"
 [(match_operand:AVX256MODE 0 "register_operand" "")
 (match_operand:AVX256MODE 1 "register_operand" "")
 (match_operand:<avxhalfvecmode> 2 "nonimmediate_operand" "")
 	    (match_operand:AVX256MODE4P 1 "register_operand" "x")
 	    (parallel [(const_int 2) (const_int 3)]))))]
 "TARGET_AVX"
 "vinsertf128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_hi_<mode>"
 [(set (match_operand:AVX256MODE4P 0 "register_operand" "=x")
 	    (parallel [(const_int 0) (const_int 1)]))
 	  (match_operand:<avxhalfvecmode> 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX"
 "vinsertf128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_lo_<mode>"
 [(set (match_operand:AVX256MODE8P 0 "register_operand" "=x")
 	    (parallel [(const_int 4) (const_int 5)
 		       (const_int 6) (const_int 7)]))))]
 "TARGET_AVX"
 "vinsertf128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_hi_<mode>"
 [(set (match_operand:AVX256MODE8P 0 "register_operand" "=x")
 		       (const_int 2) (const_int 3)]))
 	  (match_operand:<avxhalfvecmode> 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX"
 "vinsertf128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_lo_v16hi"
 [(set (match_operand:V16HI 0 "register_operand" "=x")
 		       (const_int 12) (const_int 13)
 		       (const_int 14) (const_int 15)]))))]
 "TARGET_AVX"
 "vinsertf128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_hi_v16hi"
 [(set (match_operand:V16HI 0 "register_operand" "=x")
 		       (const_int 6) (const_int 7)]))
 	  (match_operand:V8HI 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX"
 "vinsertf128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_lo_v32qi"
 [(set (match_operand:V32QI 0 "register_operand" "=x")
 		       (const_int 28) (const_int 29)
 		       (const_int 30) (const_int 31)]))))]
 "TARGET_AVX"
 "vinsertf128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "vec_set_hi_v32qi"
 [(set (match_operand:V32QI 0 "register_operand" "=x")
 		       (const_int 14) (const_int 15)]))
 	  (match_operand:V16QI 2 "nonimmediate_operand" "xm")))]
 "TARGET_AVX"
 "vinsertf128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
 [(set_attr "type" "sselog")
+(set_attr "prefix_extra" "1")
+(set_attr "length_immediate" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "V8SF")])
 (define_insn "avx_maskloadp<avxmodesuffixf2c><avxmodesuffix>"
 [(set (match_operand:AVXMODEF2P 0 "register_operand" "=x")
 	   (match_dup 0)]
 	  UNSPEC_MASKLOAD))]
 "TARGET_AVX"
 "vmaskmovp<avxmodesuffixf2c>\t{%1, %2, %0|%0, %2, %1}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_maskstorep<avxmodesuffixf2c><avxmodesuffix>"
 [(set (match_operand:AVXMODEF2P 0 "memory_operand" "=m")
 	   (match_dup 0)]
 	  UNSPEC_MASKSTORE))]
 "TARGET_AVX"
 "vmaskmovp<avxmodesuffixf2c>\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "sselog1")
+(set_attr "prefix_extra" "1")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "avx_<avxmodesuffixp><avxmodesuffix>_<avxmodesuffixp>"
 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x,x")
 default:
 gcc_unreachable ();
 }
 }
 [(set_attr "type" "sselog,ssemov")
+(set_attr "prefix_extra" "1,*")
+(set_attr "length_immediate" "1,*")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<avxvecmode>")])

Mercurial > hg > CbC > CbC_gcc

comparison gcc/config/i386/sse.md @ 55:77e2b8dfacca gcc-4.4.5