CbC/CbC_gcc: gcc/config/i386/i386.md comparison

comparison gcc/config/i386/i386.md @ 146:351920fa3827

merge

author	anatofuz <anatofuz@cr.ie.u-ryukyu.ac.jp>
date	Sun, 01 Mar 2020 16:13:28 +0900
parents	d34655255c78 1830386684a0
children

comparison

equal deleted inserted replaced

-:8f4e72ab4e11
+:351920fa3827
 ;; GCC machine description for IA-32 and x86-64.
-;; Copyright (C) 1988-2018 Free Software Foundation, Inc.
+;; Copyright (C) 1988-2020 Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
 ;;
 ;; This file is part of GCC.
 ;;
 UNSPEC_RSQRT
 UNSPEC_PSADBW
 ;; Generic math support
 UNSPEC_COPYSIGN
+UNSPEC_XORSIGN
 UNSPEC_IEEE_MIN	; not commutative
 UNSPEC_IEEE_MAX	; not commutative
 ;; x87 Floating point
 UNSPEC_SIN
 UNSPEC_F2XM1
 UNSPEC_TAN
 UNSPEC_FXAM
 ;; x87 Rounding
+UNSPEC_FRNDINT_ROUNDEVEN
 UNSPEC_FRNDINT_FLOOR
 UNSPEC_FRNDINT_CEIL
 UNSPEC_FRNDINT_TRUNC
 UNSPEC_FIST_FLOOR
 UNSPEC_FIST_CEIL
 ;; For CLDEMOTE support
 UNSPECV_CLDEMOTE
 ;; For Speculation Barrier support
 UNSPECV_SPECULATION_BARRIER
+UNSPECV_PTWRITE
+;; For ENQCMD and ENQCMDS support
+UNSPECV_ENQCMD
+UNSPECV_ENQCMDS
 ])
 ;; Constants to represent rounding modes in the ROUND instruction
 (define_constants
-[(ROUND_FLOOR			0x1)
+[(ROUND_ROUNDEVEN		0x0)
+(ROUND_FLOOR			0x1)
 (ROUND_CEIL			0x2)
 (ROUND_TRUNC			0x3)
 (ROUND_MXCSR			0x4)
 (ROUND_NO_EXC		0x8)
 ])
 ;; Processor type.
 (define_attr "cpu" "none,pentium,pentiumpro,geode,k6,athlon,k8,core2,nehalem,
 		    atom,slm,glm,haswell,generic,amdfam10,bdver1,bdver2,bdver3,
-		    bdver4,btver2,znver1"
+		    bdver4,btver2,znver1,znver2"
 (const (symbol_ref "ix86_schedule")))
 ;; A basic instruction type.  Refinements due to arguments to be
 ;; provided in other attributes.
 (define_attr "type"
 (define_attr "fp_int_src" "false,true"
 (const_string "false"))
 ;; Defines rounding mode of an FP operation.
-(define_attr "i387_cw" "trunc,floor,ceil,uninitialized,any"
+(define_attr "i387_cw" "roundeven,floor,ceil,trunc,uninitialized,any"
 (const_string "any"))
+;; Define attribute to indicate AVX insns with partial XMM register update.
+(define_attr "avx_partial_xmm_update" "false,true"
+(const_string "false"))
 ;; Define attribute to classify add/sub insns that consumes carry flag (CF)
 (define_attr "use_carry" "0,1" (const_string "0"))
 ;; Define attribute to indicate unaligned ssemov insns
 (define_attr "movu" "0,1" (const_string "0"))
 ;; Used to control the "enabled" attribute on a per-instruction basis.
 (define_attr "isa" "base,x64,x64_sse2,x64_sse4,x64_sse4_noavx,x64_avx,nox64,
-		    sse2,sse2_noavx,sse3,sse4,sse4_noavx,avx,noavx,
+		    sse_noavx,sse2,sse2_noavx,sse3,sse4,sse4_noavx,avx,noavx,
 		    avx2,noavx2,bmi,bmi2,fma4,fma,avx512f,noavx512f,
 		    avx512bw,noavx512bw,avx512dq,noavx512dq,
 		    avx512vl,noavx512vl,x64_avx512dq,x64_avx512bw"
+(const_string "base"))
+;; Define instruction set of MMX instructions
+(define_attr "mmx_isa" "base,native,sse,sse_noavx,avx"
 (const_string "base"))
 (define_attr "enabled" ""
 (cond [(eq_attr "isa" "x64") (symbol_ref "TARGET_64BIT")
 	 (eq_attr "isa" "x64_sse2")
 	   (symbol_ref "TARGET_64BIT && TARGET_AVX512DQ")
 	 (eq_attr "isa" "x64_avx512bw")
 	   (symbol_ref "TARGET_64BIT && TARGET_AVX512BW")
 	 (eq_attr "isa" "nox64") (symbol_ref "!TARGET_64BIT")
 	 (eq_attr "isa" "sse2") (symbol_ref "TARGET_SSE2")
+	 (eq_attr "isa" "sse_noavx")
+	   (symbol_ref "TARGET_SSE && !TARGET_AVX")
 	 (eq_attr "isa" "sse2_noavx")
 	   (symbol_ref "TARGET_SSE2 && !TARGET_AVX")
 	 (eq_attr "isa" "sse3") (symbol_ref "TARGET_SSE3")
 	 (eq_attr "isa" "sse4") (symbol_ref "TARGET_SSE4_1")
 	 (eq_attr "isa" "sse4_noavx")
 	 (eq_attr "isa" "noavx512bw") (symbol_ref "!TARGET_AVX512BW")
 	 (eq_attr "isa" "avx512dq") (symbol_ref "TARGET_AVX512DQ")
 	 (eq_attr "isa" "noavx512dq") (symbol_ref "!TARGET_AVX512DQ")
 	 (eq_attr "isa" "avx512vl") (symbol_ref "TARGET_AVX512VL")
 	 (eq_attr "isa" "noavx512vl") (symbol_ref "!TARGET_AVX512VL")
+	 (eq_attr "mmx_isa" "native")
+	   (symbol_ref "!TARGET_MMX_WITH_SSE")
+	 (eq_attr "mmx_isa" "sse")
+	   (symbol_ref "TARGET_MMX_WITH_SSE")
+	 (eq_attr "mmx_isa" "sse_noavx")
+	   (symbol_ref "TARGET_MMX_WITH_SSE && !TARGET_AVX")
+	 (eq_attr "mmx_isa" "avx")
+	   (symbol_ref "TARGET_MMX_WITH_SSE && TARGET_AVX")
 	]
 	(const_int 1)))
 (define_attr "preferred_for_size" "" (const_int 1))
 (define_attr "preferred_for_speed" "" (const_int 1))
 ;; Used in signed and unsigned widening multiplications.
 (define_code_iterator any_extend [sign_extend zero_extend])
 ;; Prefix for insn menmonic.
-(define_code_attr sgnprefix [(sign_extend "i") (zero_extend "")])
+(define_code_attr sgnprefix [(sign_extend "i") (zero_extend "")
+			     (div "i") (udiv "")])
 ;; Prefix for define_insn
-(define_code_attr u [(sign_extend "") (zero_extend "u")])
 (define_code_attr s [(sign_extend "s") (zero_extend "u")])
-(define_code_attr u_bool [(sign_extend "false") (zero_extend "true")])
+(define_code_attr u [(sign_extend "") (zero_extend "u")
+		     (div "") (udiv "u")])
+(define_code_attr u_bool [(sign_extend "false") (zero_extend "true")
+			  (div "false") (udiv "true")])
 ;; Used in signed and unsigned truncations.
 (define_code_iterator any_truncate [ss_truncate truncate us_truncate])
 ;; Instruction suffix for truncations.
 (define_code_attr trunsuffix [(ss_truncate "s") (truncate "") (us_truncate "us")])
 ;; Math-dependant integer modes without DImode.
 (define_mode_iterator SWIM124 [(QI "TARGET_QIMODE_MATH")
 			       (HI "TARGET_HIMODE_MATH")
 			       SI])
-;; Math-dependant integer modes with DImode.
+;; Math-dependant integer modes with DImode (enabled for 32bit with STV).
-(define_mode_iterator SWIM1248x [(QI "TARGET_QIMODE_MATH")
+(define_mode_iterator SWIM1248s
-				 (HI "TARGET_HIMODE_MATH")
+	[(QI "TARGET_QIMODE_MATH")
-				 SI (DI "(TARGET_STV && TARGET_SSE2) || TARGET_64BIT")])
+	 (HI "TARGET_HIMODE_MATH")
+	 SI (DI "TARGET_64BIT || (TARGET_STV && TARGET_SSE2)")])
 ;; Math-dependant single word integer modes without QImode.
 (define_mode_iterator SWIM248 [(HI "TARGET_HIMODE_MATH")
 		      	       SI (DI "TARGET_64BIT")])
 ;; Double word integer modes.
 (define_mode_iterator DWI [(DI "!TARGET_64BIT")
 			   (TI "TARGET_64BIT")])
+;; SWI and DWI together.
+(define_mode_iterator SWIDWI [QI HI SI DI (TI "TARGET_64BIT")])
 ;; GET_MODE_SIZE for selected modes.  As GET_MODE_SIZE is not
 ;; compile time constant, it is faster to use <MODE_SIZE> than
 ;; GET_MODE_SIZE (<MODE>mode).  For XFmode which depends on
 ;; command line options just use GET_MODE_SIZE macro.
 			     (V1TI "16") (V2TI "32") (V4TI "64")
 			     (V2DF "16") (V4DF "32") (V8DF "64")
 			     (V4SF "16") (V8SF "32") (V16SF "64")])
 ;; Double word integer modes as mode attribute.
-(define_mode_attr DWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI")])
+(define_mode_attr DWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI") (TI "OI")])
-(define_mode_attr dwi [(QI "hi") (HI "si") (SI "di") (DI "ti")])
+(define_mode_attr dwi [(QI "hi") (HI "si") (SI "di") (DI "ti") (TI "oi")])
 ;; LEA mode corresponding to an integer mode
 (define_mode_attr LEAMODE [(QI "SI") (HI "SI") (SI "SI") (DI "DI")])
 ;; Half mode for double word integer modes.
 	[(QI "general_operand")
 	 (HI "general_operand")
 	 (SI "x86_64_szext_general_operand")
 	 (DI "x86_64_szext_general_operand")])
+(define_mode_attr nonmemory_szext_operand
+	[(QI "nonmemory_operand")
+	 (HI "nonmemory_operand")
+	 (SI "x86_64_szext_nonmemory_operand")
+	 (DI "x86_64_szext_nonmemory_operand")])
 ;; Immediate operand predicate for integer modes.
 (define_mode_attr immediate_operand
 	[(QI "immediate_operand")
 	 (HI "immediate_operand")
 	 (SI "x86_64_immediate_operand")
 ;; SSE and x87 SFmode and DFmode floating point modes
 (define_mode_iterator MODEF [SF DF])
 ;; All x87 floating point modes
 (define_mode_iterator X87MODEF [SF DF XF])
+;; All SSE floating point modes
+(define_mode_iterator SSEMODEF [SF DF TF])
+(define_mode_attr ssevecmodef [(SF "V4SF") (DF "V2DF") (TF "TF")])
 ;; SSE instruction suffix for various modes
 (define_mode_attr ssemodesuffix
 [(SF "ss") (DF "sd")
 (V16SF "ps") (V8DF "pd")
 ;; AVX512F vector mode corresponding to a scalar mode
 (define_mode_attr avx512fvecmode
 [(QI "V64QI") (HI "V32HI") (SI "V16SI") (DI "V8DI") (SF "V16SF") (DF "V8DF")])
 ;; Instruction suffix for REX 64bit operators.
-(define_mode_attr rex64suffix [(SI "") (DI "{q}")])
+(define_mode_attr rex64suffix [(SI "{l}") (DI "{q}")])
 (define_mode_attr rex64namesuffix [(SI "") (DI "q")])
 ;; This mode iterator allows :P to be used for patterns that operate on
 ;; pointer-sized quantities.  Exactly one of the two alternatives will match.
 (define_mode_iterator P [(SI "Pmode == SImode") (DI "Pmode == DImode")])
 ix86_expand_setcc (operands[0], GET_CODE (operands[1]),
 		     operands[2], operands[3]);
 DONE;
 })
-(define_expand "cmp<mode>_1"
+(define_expand "@cmp<mode>_1"
 [(set (reg:CC FLAGS_REG)
 	(compare:CC (match_operand:SWI48 0 "nonimmediate_operand")
 		    (match_operand:SWI48 1 "<general_operand>")))])
-(define_mode_iterator SWI1248_AVX512BWDQ2_64
+(define_mode_iterator SWI1248_AVX512BWDQ_64
-[(QI "TARGET_AVX512DQ") (HI "TARGET_AVX512DQ")
+[(QI "TARGET_AVX512DQ") HI
 (SI "TARGET_AVX512BW") (DI "TARGET_AVX512BW && TARGET_64BIT")])
 (define_insn "*cmp<mode>_ccz_1"
 [(set (reg FLAGS_REG)
-	(compare (match_operand:SWI1248_AVX512BWDQ2_64 0
+	(compare (match_operand:SWI1248_AVX512BWDQ_64 0
 			"nonimmediate_operand" "<r>,?m<r>,$k")
-		 (match_operand:SWI1248_AVX512BWDQ2_64 1 "const0_operand")))]
+		 (match_operand:SWI1248_AVX512BWDQ_64 1 "const0_operand")))]
-"ix86_match_ccmode (insn, CCZmode)"
+"TARGET_AVX512F && ix86_match_ccmode (insn, CCZmode)"
 "@
 test{<imodesuffix>}\t%0, %0
 cmp{<imodesuffix>}\t{%1, %0|%0, %1}
-ktest<mskmodesuffix>\t%0, %0"
+kortest<mskmodesuffix>\t%0, %0"
 [(set_attr "type" "test,icmp,msklog")
 (set_attr "length_immediate" "0,1,*")
 (set_attr "prefix" "*,*,vex")
 (set_attr "mode" "<MODE>")])
 ;; Convert impossible pushes of immediate to existing instructions.
 ;; First try to get scratch register and go through it.  In case this
 ;; fails, push sign extended lower part first and then overwrite
 ;; upper part by 32bit move.
 (define_peephole2
 [(match_scratch:DI 2 "r")
 (set (match_operand:DI 0 "push_operand")
 (match_operand:DI 1 "immediate_operand"))]
-"TARGET_64BIT && !symbolic_operand (operands[1], DImode)
+"TARGET_64BIT
+&& !symbolic_operand (operands[1], DImode)
 && !x86_64_immediate_operand (operands[1], DImode)"
 [(set (match_dup 2) (match_dup 1))
 (set (match_dup 0) (match_dup 2))])
-;; We need to define this as both peepholer and splitter for case
-;; peephole2 pass is not run.
-;; "&& 1" is needed to keep it from matching the previous pattern.
-(define_peephole2
-[(set (match_operand:DI 0 "push_operand")
-(match_operand:DI 1 "immediate_operand"))]
-"TARGET_64BIT && !symbolic_operand (operands[1], DImode)
-&& !x86_64_immediate_operand (operands[1], DImode) && 1"
-[(set (match_dup 0) (match_dup 1))
-(set (match_dup 2) (match_dup 3))]
-{
-split_double_mode (DImode, &operands[1], 1, &operands[2], &operands[3]);
-operands[1] = gen_lowpart (DImode, operands[2]);
-operands[2] = gen_rtx_MEM (SImode, gen_rtx_PLUS (Pmode, stack_pointer_rtx,
-						   GEN_INT (4)));
-})
 (define_split
 [(set (match_operand:DI 0 "push_operand")
 (match_operand:DI 1 "immediate_operand"))]
-"TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+"TARGET_64BIT && epilogue_completed
-		    ? epilogue_completed : reload_completed)
 && !symbolic_operand (operands[1], DImode)
 && !x86_64_immediate_operand (operands[1], DImode)"
 [(set (match_dup 0) (match_dup 1))
 (set (match_dup 2) (match_dup 3))]
 {
 case TYPE_SSEMOV:
 if (misaligned_operand (operands[0], OImode)
 	  || misaligned_operand (operands[1], OImode))
 	{
-	  if (get_attr_mode (insn) == MODE_V8SF)
+	  if (get_attr_mode (insn) == MODE_XI)
-	    return "vmovups\t{%1, %0|%0, %1}";
-	  else if (get_attr_mode (insn) == MODE_XI)
 	    return "vmovdqu32\t{%1, %0|%0, %1}";
 	  else
 	    return "vmovdqu\t{%1, %0|%0, %1}";
 	}
 else
 	{
-	  if (get_attr_mode (insn) == MODE_V8SF)
+	  if (get_attr_mode (insn) == MODE_XI)
-	    return "vmovaps\t{%1, %0|%0, %1}";
-	  else if (get_attr_mode (insn) == MODE_XI)
 	    return "vmovdqa32\t{%1, %0|%0, %1}";
 	  else
 	    return "vmovdqa\t{%1, %0|%0, %1}";
 	}
 		    (match_operand 1 "ext_sse_reg_operand"))
 		 (const_string "XI")
 	       (and (eq_attr "alternative" "1")
 		    (match_test "TARGET_AVX512VL"))
 		 (const_string "XI")
-	       (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
-		    (and (eq_attr "alternative" "3")
-			 (match_test "TARGET_SSE_TYPELESS_STORES")))
-		 (const_string "V8SF")
 	      ]
 	      (const_string "OI")))])
 (define_insn "*movti_internal"
 [(set (match_operand:TI 0 "nonimmediate_operand" "=!r ,o ,v,v ,v ,m,?r,?Yd")
 		    (match_operand 1 "ext_sse_reg_operand"))
 		 (const_string "XI")
 	       (and (eq_attr "alternative" "3")
 		    (match_test "TARGET_AVX512VL"))
 		 (const_string "XI")
-	       (ior (not (match_test "TARGET_SSE2"))
-		    (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
-			 (and (eq_attr "alternative" "5")
-			      (match_test "TARGET_SSE_TYPELESS_STORES"))))
-		 (const_string "V4SF")
 	       (match_test "TARGET_AVX")
 		 (const_string "TI")
-	       (match_test "optimize_function_for_size_p (cfun)")
+	       (ior (not (match_test "TARGET_SSE2"))
+		    (match_test "optimize_function_for_size_p (cfun)"))
+		 (const_string "V4SF")
+	       (and (eq_attr "alternative" "5")
+		    (match_test "TARGET_SSE_TYPELESS_STORES"))
 		 (const_string "V4SF")
 	       ]
 	       (const_string "TI")))
 (set (attr "preferred_for_speed")
 (cond [(eq_attr "alternative" "6")
 		  gen_lowpart (DImode, operands[1]));
 })
 (define_insn "*movdi_internal"
 [(set (match_operand:DI 0 "nonimmediate_operand"
-"=r  ,o  ,r,r  ,r,m ,*y,*y,?*y,?m,?r,?*y,*v,*v,*v,m ,m,?r ,?*Yd,?r,?*v,?*y,?*x,*k,*k ,*r,*m")
+"=r  ,o  ,r,r  ,r,m ,*y,*y,?*y,?m,?r,?*y,*v,*v,*v,m ,m,?r ,?*Yd,?r,?*v,?*y,?*x,*k,*k ,*r,*m,*k")
 	(match_operand:DI 1 "general_operand"
-"riFo,riF,Z,rem,i,re,C ,*y,m  ,*y,*y,r  ,C ,*v,m ,*v,v,*Yd,r   ,*v,r  ,*x ,*y ,*r,*km,*k,*k"))]
+"riFo,riF,Z,rem,i,re,C ,*y,m  ,*y,*y,r  ,C ,*v,m ,*v,v,*Yd,r   ,*v,r  ,*x ,*y ,*r,*km,*k,*k,CBC"))]
 "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
 switch (get_attr_type (insn))
 {
 case TYPE_MSKMOV:
 return "kmovq\t{%1, %0|%0, %1}";
+case TYPE_MSKLOG:
+if (operands[1] == const0_rtx)
+	return "kxorq\t%0, %0, %0";
+else if (operands[1] == constm1_rtx)
+	return "kxnorq\t%0, %0, %0";
+gcc_unreachable ();
 case TYPE_MULTI:
 return "#";
 case TYPE_MMX:
 	      (const_string "ssemov")
 	    (eq_attr "alternative" "21,22")
 	      (const_string "ssecvt")
 	    (eq_attr "alternative" "23,24,25,26")
 	      (const_string "mskmov")
+	    (eq_attr "alternative" "27")
+	      (const_string "msklog")
 	    (and (match_operand 0 "register_operand")
 		 (match_operand 1 "pic_32bit_operand"))
 	      (const_string "lea")
 	   ]
 	   (const_string "imov")))
 	      (const_string "SI")
 	    (eq_attr "alternative" "12,13")
 	      (cond [(ior (match_operand 0 "ext_sse_reg_operand")
 			  (match_operand 1 "ext_sse_reg_operand"))
 		       (const_string "TI")
-		     (ior (not (match_test "TARGET_SSE2"))
-			  (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
-		       (const_string "V4SF")
 		     (match_test "TARGET_AVX")
 		       (const_string "TI")
-		     (match_test "optimize_function_for_size_p (cfun)")
+		     (ior (not (match_test "TARGET_SSE2"))
+			  (match_test "optimize_function_for_size_p (cfun)"))
 		       (const_string "V4SF")
 		    ]
 		    (const_string "TI"))
 	    (and (eq_attr "alternative" "14,15,16")
 operands[2] = gen_int_mode (shift, QImode);
 })
 (define_insn "*movsi_internal"
 [(set (match_operand:SI 0 "nonimmediate_operand"
-"=r,m ,*y,*y,?*y,?m,?r,?*y,*v,*v,*v,m ,?r,?*v,*k,*k ,*rm")
+"=r,m ,*y,*y,?*y,?m,?r,?*y,*v,*v,*v,m ,?r,?*v,*k,*k ,*rm,*k")
 	(match_operand:SI 1 "general_operand"
-"g ,re,C ,*y,m  ,*y,*y,r  ,C ,*v,m ,*v,*v,r  ,*r,*km,*k"))]
+"g ,re,C ,*y,m  ,*y,*y,r  ,C ,*v,m ,*v,*v,r  ,*r,*km,*k ,CBC"))]
 "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
 switch (get_attr_type (insn))
 {
 case TYPE_SSELOG1:
 return standard_sse_constant_opcode (insn, operands);
 case TYPE_MSKMOV:
 return "kmovd\t{%1, %0|%0, %1}";
+case TYPE_MSKLOG:
+if (operands[1] == const0_rtx)
+	return "kxord\t%0, %0, %0";
+else if (operands[1] == constm1_rtx)
+	return "kxnord\t%0, %0, %0";
+gcc_unreachable ();
 case TYPE_SSEMOV:
 switch (get_attr_mode (insn))
 	{
 	case MODE_SI:
 	      (const_string "sselog1")
 	    (eq_attr "alternative" "9,10,11,12,13")
 	      (const_string "ssemov")
 	    (eq_attr "alternative" "14,15,16")
 	      (const_string "mskmov")
+	    (eq_attr "alternative" "17")
+	      (const_string "msklog")
 	    (and (match_operand 0 "register_operand")
 		 (match_operand 1 "pic_32bit_operand"))
 	      (const_string "lea")
 	   ]
 	   (const_string "imov")))
 	      (const_string "DI")
 	    (eq_attr "alternative" "8,9")
 	      (cond [(ior (match_operand 0 "ext_sse_reg_operand")
 			  (match_operand 1 "ext_sse_reg_operand"))
 		       (const_string "XI")
-		     (ior (not (match_test "TARGET_SSE2"))
-			  (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
-		       (const_string "V4SF")
 		     (match_test "TARGET_AVX")
 		       (const_string "TI")
-		     (match_test "optimize_function_for_size_p (cfun)")
+		     (ior (not (match_test "TARGET_SSE2"))
+			  (match_test "optimize_function_for_size_p (cfun)"))
 		       (const_string "V4SF")
 		    ]
 		    (const_string "TI"))
 	    (and (eq_attr "alternative" "10,11")
 	      (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC")
 	   ]
 	   (symbol_ref "true")))])
 (define_insn "*movhi_internal"
-[(set (match_operand:HI 0 "nonimmediate_operand" "=r,r ,r ,m ,k,k ,r,m")
+[(set (match_operand:HI 0 "nonimmediate_operand" "=r,r ,r ,m ,k,k ,r,m,k")
-	(match_operand:HI 1 "general_operand"      "r ,rn,rm,rn,r,km,k,k"))]
+	(match_operand:HI 1 "general_operand"      "r ,rn,rm,rn,r,km,k,k,CBC"))]
 "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
 switch (get_attr_type (insn))
 {
 case TYPE_IMOVX:
 	  return "kmovw\t{%1, %0|%0, %1}";
 	default:
 	  gcc_unreachable ();
 	}
+case TYPE_MSKLOG:
+if (operands[1] == const0_rtx)
+	return "kxorw\t%0, %0, %0";
+else if (operands[1] == constm1_rtx)
+	return "kxnorw\t%0, %0, %0";
+gcc_unreachable ();
 default:
 if (get_attr_mode (insn) == MODE_SI)
 	return "mov{l}\t{%k1, %k0|%k0, %k1}";
 else
 	return "mov{w}\t{%1, %0|%0, %1}";
 }
 }
 [(set (attr "type")
 (cond [(eq_attr "alternative" "4,5,6,7")
 	      (const_string "mskmov")
+	    (eq_attr "alternative" "8")
+	      (const_string "msklog")
 	    (match_test "optimize_function_for_size_p (cfun)")
 	      (const_string "imov")
 	    (and (eq_attr "alternative" "0")
 		 (ior (not (match_test "TARGET_PARTIAL_REG_STALL"))
 		      (not (match_test "TARGET_HIMODE_MATH"))))
 		 (eq_attr "alternative" "0,2"))
 	      (const_string "imovx")
 	   ]
 	   (const_string "imov")))
 (set (attr "prefix")
-(if_then_else (eq_attr "alternative" "4,5,6,7")
+(if_then_else (eq_attr "alternative" "4,5,6,7,8")
 	(const_string "vex")
 	(const_string "orig")))
 (set (attr "mode")
 (cond [(eq_attr "type" "imovx")
 	       (const_string "SI")
 ;; register stall machines with, where we use QImode instructions, since
 ;; partial register stall can be caused there.  Then we use movzx.
 (define_insn "*movqi_internal"
 [(set (match_operand:QI 0 "nonimmediate_operand"
-			"=Q,R,r,q,q,r,r ,?r,m ,k,k,r,m,k")
+			"=Q,R,r,q,q,r,r ,?r,m ,k,k,r,m,k,k,k")
 	(match_operand:QI 1 "general_operand"
-			"Q ,R,r,n,m,q,rn, m,qn,r,k,k,k,m"))]
+			"Q ,R,r,n,m,q,rn, m,qn,r,k,k,k,m,C,BC"))]
 "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
-static char buf[128];
+char buf[128];
 const char *ops;
 const char *suffix;
 switch (get_attr_type (insn))
 {
 	}
 suffix = (get_attr_mode (insn) == MODE_HI) ? "w" : "b";
 snprintf (buf, sizeof (buf), ops, suffix);
-return buf;
+output_asm_insn (buf, operands);
+return "";
+case TYPE_MSKLOG:
+if (operands[1] == const0_rtx)
+	{
+	  if (get_attr_mode (insn) == MODE_HI)
+	    return "kxorw\t%0, %0, %0";
+	  else
+	    return "kxorb\t%0, %0, %0";
+	}
+else if (operands[1] == constm1_rtx)
+	{
+	  gcc_assert (TARGET_AVX512DQ);
+	  return "kxnorb\t%0, %0, %0";
+	}
+gcc_unreachable ();
 default:
 if (get_attr_mode (insn) == MODE_SI)
 	return "mov{l}\t{%k1, %k0|%k0, %k1}";
 else
 }
 }
 [(set (attr "isa")
 (cond [(eq_attr "alternative" "1,2")
 	      (const_string "x64")
-	    (eq_attr "alternative" "12,13")
+	    (eq_attr "alternative" "12,13,15")
 	      (const_string "avx512dq")
 	   ]
 	   (const_string "*")))
 (set (attr "type")
 (cond [(eq_attr "alternative" "9,10,11,12,13")
 	      (const_string "mskmov")
+	    (eq_attr "alternative" "14,15")
+	      (const_string "msklog")
 	    (and (eq_attr "alternative" "7")
 		 (not (match_operand:QI 1 "aligned_operand")))
 	      (const_string "imovx")
 	    (match_test "optimize_function_for_size_p (cfun)")
 	      (const_string "imov")
 		 (eq_attr "alternative" "4"))
 	      (const_string "imovx")
 	   ]
 	   (const_string "imov")))
 (set (attr "prefix")
-(if_then_else (eq_attr "alternative" "9,10,11")
+(if_then_else (eq_attr "alternative" "9,10,11,12,13,14,15")
 (const_string "vex")
 (const_string "orig")))
 (set (attr "mode")
 (cond [(eq_attr "alternative" "5,6,7")
 	       (const_string "SI")
 	     (eq_attr "alternative" "8")
 	       (const_string "QI")
-	     (and (eq_attr "alternative" "9,10,11")
+	     (and (eq_attr "alternative" "9,10,11,14")
 		  (not (match_test "TARGET_AVX512DQ")))
 	       (const_string "HI")
 	     (eq_attr "type" "imovx")
 	       (const_string "SI")
 	     ;; For -Os, 8-bit immediates are always shorter than 32-bit
 (set_attr "pent_pair" "np")
 (set_attr "athlon_decode" "vector")
 (set_attr "amdfam10_decode" "double")
 (set_attr "bdver1_decode" "double")])
+(define_peephole2
+[(set (match_operand:SWI 0 "general_reg_operand")
+	(match_operand:SWI 1 "general_reg_operand"))
+(set (match_dup 1)
+	(match_operand:SWI 2 "general_reg_operand"))
+(set (match_dup 2) (match_dup 0))]
+"peep2_reg_dead_p (3, operands[0])
+&& optimize_insn_for_size_p ()"
+[(parallel [(set (match_dup 1) (match_dup 2))
+	      (set (match_dup 2) (match_dup 1))])])
 (define_expand "movstrict<mode>"
-[(set (strict_low_part (match_operand:SWI12 0 "nonimmediate_operand"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand"))
 	(match_operand:SWI12 1 "general_operand"))]
 ""
 {
-if (TARGET_PARTIAL_REG_STALL && optimize_function_for_speed_p (cfun))
+gcc_assert (SUBREG_P (operands[0]));
+if ((TARGET_PARTIAL_REG_STALL && optimize_function_for_speed_p (cfun))
+|| GET_MODE_CLASS (GET_MODE (SUBREG_REG (operands[0]))) != MODE_INT)
 FAIL;
-if (SUBREG_P (operands[0])
-&& GET_MODE_CLASS (GET_MODE (SUBREG_REG (operands[0]))) != MODE_INT)
-FAIL;
-/* Don't generate memory->memory moves, go through a register */
-if (MEM_P (operands[0]) && MEM_P (operands[1]))
-operands[1] = force_reg (<MODE>mode, operands[1]);
 })
 (define_insn "*movstrict<mode>_1"
 [(set (strict_low_part
-	  (match_operand:SWI12 0 "nonimmediate_operand" "+<r>m,<r>"))
+	  (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(match_operand:SWI12 1 "general_operand" "<r>n,m"))]
+	(match_operand:SWI12 1 "general_operand" "<r>mn"))]
-"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+"!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "mov{<imodesuffix>}\t{%1, %0|%0, %1}"
 [(set_attr "type" "imov")
 (set_attr "mode" "<MODE>")])
 (define_insn "*movstrict<mode>_xor"
 [(set (match_operand:TF 0 "push_operand" "=<,<")
 	(match_operand:TF 1 "general_no_elim_operand" "v,*roC"))]
 "TARGET_64BIT || TARGET_SSE"
 {
 /* This insn should be already split before reg-stack.  */
-gcc_unreachable ();
+return "#";
 }
 [(set_attr "isa" "*,x64")
 (set_attr "type" "multi")
 (set_attr "unit" "sse,*")
 (set_attr "mode" "TF,DI")])
 [(set (match_operand:XF 0 "push_operand" "=<,<,<,<,<")
 	(match_operand:XF 1 "general_no_elim_operand" "f,r,*r,oF,oC"))]
 ""
 {
 /* This insn should be already split before reg-stack.  */
-gcc_unreachable ();
+return "#";
 }
 [(set_attr "isa" "*,*,*,nox64,x64")
 (set_attr "type" "multi")
 (set_attr "unit" "i387,*,*,*,*")
 (set (attr "mode")
 [(set (match_operand:DF 0 "push_operand" "=<,<,<,<,<,<")
 	(match_operand:DF 1 "general_no_elim_operand" "f,r,*r,oF,rmC,x"))]
 ""
 {
 /* This insn should be already split before reg-stack.  */
-gcc_unreachable ();
+return "#";
 }
 [(set_attr "isa" "*,nox64,nox64,nox64,x64,sse2")
 (set_attr "type" "multi")
 (set_attr "unit" "i387,*,*,*,*,sse")
 (set_attr "mode" "DF,SI,SI,SI,DI,DF")
 [(set (match_operand:SF 0 "push_operand" "=X,X,X")
 	(match_operand:SF 1 "nonmemory_no_elim_operand" "f,rF,x"))]
 "TARGET_64BIT"
 {
 /* Anything else should be already split before reg-stack.  */
-gcc_assert (which_alternative == 1);
+if (which_alternative != 1)
+return "#";
 return "push{q}\t%q1";
 }
 [(set_attr "type" "multi,push,multi")
 (set_attr "unit" "i387,*,*")
 (set_attr "mode" "SF,DI,SF")])
 [(set (match_operand:SF 0 "push_operand" "=<,<,<")
 	(match_operand:SF 1 "general_no_elim_operand" "f,rmF,x"))]
 "!TARGET_64BIT"
 {
 /* Anything else should be already split before reg-stack.  */
-gcc_assert (which_alternative == 1);
+if (which_alternative != 1)
+return "#";
 return "push{l}\t%1";
 }
 [(set_attr "type" "multi,push,multi")
 (set_attr "unit" "i387,*,*")
 (set_attr "mode" "SF,SI,SF")])
 (const_string "maybe_vex")
 (const_string "orig")))
 (set (attr "mode")
 (cond [(eq_attr "alternative" "3,4")
 		 (const_string "DI")
+	       (match_test "TARGET_AVX")
+		 (const_string "TI")
+	       (ior (not (match_test "TARGET_SSE2"))
+		    (match_test "optimize_function_for_size_p (cfun)"))
+		 (const_string "V4SF")
 	       (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
 		 (const_string "V4SF")
 	       (and (eq_attr "alternative" "2")
 		    (match_test "TARGET_SSE_TYPELESS_STORES"))
-		 (const_string "V4SF")
-	       (match_test "TARGET_AVX")
-		 (const_string "TI")
-	       (ior (not (match_test "TARGET_SSE2"))
-		    (match_test "optimize_function_for_size_p (cfun)"))
 		 (const_string "V4SF")
 	       ]
 	       (const_string "TI")))])
 (define_split
 	       (eq_attr "alternative" "8,9,11,20,21,24,25")
 		 (const_string "DI")
 	       /* xorps is one byte shorter for non-AVX targets.  */
 	       (eq_attr "alternative" "12,16")
-		 (cond [(not (match_test "TARGET_SSE2"))
+		 (cond [(and (match_test "TARGET_AVX512F")
-		 	  (const_string "V4SF")
+			     (not (match_test "TARGET_PREFER_AVX256")))
-			(and (match_test "TARGET_AVX512F")
-			  (not (match_test "TARGET_PREFER_AVX256")))
 			  (const_string "XI")
 			(match_test "TARGET_AVX")
 			  (const_string "V2DF")
-			(match_test "optimize_function_for_size_p (cfun)")
+			(ior (not (match_test "TARGET_SSE2"))
+			     (match_test "optimize_function_for_size_p (cfun)"))
 			  (const_string "V4SF")
 			(match_test "TARGET_SSE_LOAD0_BY_PXOR")
 			  (const_string "TI")
 		       ]
 		       (const_string "V2DF"))
 		 (cond [(and (ior (not (match_test "TARGET_PREFER_AVX256"))
 				  (not (match_test "TARGET_AVX512VL")))
 			     (ior (match_operand 0 "ext_sse_reg_operand")
 				  (match_operand 1 "ext_sse_reg_operand")))
 			  (const_string "V8DF")
+			(match_test "TARGET_AVX")
+			  (const_string "DF")
 			(ior (not (match_test "TARGET_SSE2"))
-			     (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
+			     (match_test "optimize_function_for_size_p (cfun)"))
+			  (const_string "V4SF")
+			(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
 			  (const_string "V4SF")
 			(match_test "TARGET_SSE_PARTIAL_REG_DEPENDENCY")
 			  (const_string "V2DF")
-			(match_test "TARGET_AVX")
-			  (const_string "DF")
-			(match_test "optimize_function_for_size_p (cfun)")
-			  (const_string "V4SF")
 		       ]
 		       (const_string "DF"))
 	       /* For architectures resolving dependencies on register
 		  parts we may avoid extra work to zero out upper part
 default:
 gcc_unreachable ();
 }
 }
 [(set (attr "isa")
-(cond [(eq_attr "alternative" "14,15")
+(cond [(eq_attr "alternative" "9,10")
 	      (const_string "sse2")
 	   ]
 	   (const_string "*")))
 (set (attr "type")
 	(cond [(eq_attr "alternative" "0,1,2")
 (cond [(eq_attr "alternative" "3,4,9,10,12,13,14,15,16,17")
 		 (const_string "SI")
 	       (eq_attr "alternative" "11")
 		 (const_string "DI")
 	       (eq_attr "alternative" "5")
-		 (cond [(not (match_test "TARGET_SSE2"))
+		 (cond [(and (match_test "TARGET_AVX512F")
-		 	  (const_string "V4SF")
+			     (not (match_test "TARGET_PREFER_AVX256")))
-			(and (match_test "TARGET_AVX512F")
-			  (not (match_test "TARGET_PREFER_AVX256")))
 			  (const_string "V16SF")
 			(match_test "TARGET_AVX")
 			  (const_string "V4SF")
-			(match_test "optimize_function_for_size_p (cfun)")
+			(ior (not (match_test "TARGET_SSE2"))
+			     (match_test "optimize_function_for_size_p (cfun)"))
 			  (const_string "V4SF")
 			(match_test "TARGET_SSE_LOAD0_BY_PXOR")
 			  (const_string "TI")
 		       ]
 		       (const_string "V4SF"))
 	       /* For architectures resolving dependencies on
 [(set (match_operand:DI 0 "nonimmediate_operand")
 	(zero_extend:DI (match_operand:SI 1 "nonimmediate_operand")))])
 (define_insn "*zero_extendsidi2"
 [(set (match_operand:DI 0 "nonimmediate_operand"
-		"=r,?r,?o,r   ,o,?*y,?!*y,$r,$v,$x,*x,*v,*r")
+		"=r,?r,?o,r   ,o,?*y,?!*y,$r,$v,$x,*x,*v,*r,*k")
 	(zero_extend:DI
 	 (match_operand:SI 1 "x86_64_zext_operand"
-	        "0 ,rm,r ,rmWz,0,r  ,m   ,v ,r ,m ,*x,*v,*k")))]
+	        "0 ,rm,r ,rmWz,0,r  ,m   ,v ,r ,m ,*x,*v,*k,*km")))]
 ""
 {
 switch (get_attr_type (insn))
 {
 case TYPE_IMOVX:
 	      (const_string "sse4")
 	    (eq_attr "alternative" "11")
 	      (const_string "avx512f")
 	    (eq_attr "alternative" "12")
 	      (const_string "x64_avx512bw")
+	    (eq_attr "alternative" "13")
+	      (const_string "avx512bw")
 	   ]
 	   (const_string "*")))
+(set (attr "mmx_isa")
+(if_then_else (eq_attr "alternative" "5,6")
+		   (const_string "native")
+		   (const_string "*")))
 (set (attr "type")
 (cond [(eq_attr "alternative" "0,1,2,4")
 	      (const_string "multi")
 	    (eq_attr "alternative" "5,6")
 	      (const_string "mmxmov")
 	      (if_then_else (match_test "TARGET_64BIT")
 		(const_string "ssemov")
 		(const_string "multi"))
 	    (eq_attr "alternative" "8,9,10,11")
 	      (const_string "ssemov")
-	    (eq_attr "alternative" "12")
+	    (eq_attr "alternative" "12,13")
 	      (const_string "mskmov")
 	   ]
 	   (const_string "imovx")))
 (set (attr "prefix_extra")
 (if_then_else (eq_attr "alternative" "10,11")
 (define_mode_attr kmov_isa
 [(QI "avx512dq") (HI "avx512f") (SI "avx512bw") (DI "avx512bw")])
 (define_insn "zero_extend<mode>di2"
-[(set (match_operand:DI 0 "register_operand" "=r,*r")
+[(set (match_operand:DI 0 "register_operand" "=r,*r,*k")
 	(zero_extend:DI
-	 (match_operand:SWI12 1 "nonimmediate_operand" "<r>m,*k")))]
+	 (match_operand:SWI12 1 "nonimmediate_operand" "<r>m,*k,*km")))]
 "TARGET_64BIT"
 "@
 movz{<imodesuffix>l|x}\t{%1, %k0|%k0, %1}
+kmov<mskmodesuffix>\t{%1, %k0|%k0, %1}
 kmov<mskmodesuffix>\t{%1, %k0|%k0, %1}"
-[(set_attr "isa" "*,<kmov_isa>")
+[(set_attr "isa" "*,<kmov_isa>,<kmov_isa>")
-(set_attr "type" "imovx,mskmov")
+(set_attr "type" "imovx,mskmov,mskmov")
-(set_attr "mode" "SI,<MODE>")])
+(set_attr "mode" "SI,<MODE>,<MODE>")])
 (define_expand "zero_extend<mode>si2"
 [(set (match_operand:SI 0 "register_operand")
 	(zero_extend:SI (match_operand:SWI12 1 "nonimmediate_operand")))]
 ""
 || REGNO (operands[0]) != REGNO (operands[1]))
 {
 ix86_expand_clear (operands[0]);
 gcc_assert (!TARGET_PARTIAL_REG_STALL);
-emit_insn (gen_movstrict<mode>
+emit_insn (gen_rtx_SET
-		  (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
+		 (gen_rtx_STRICT_LOW_PART
+		  (VOIDmode, gen_lowpart (<MODE>mode, operands[0])),
+		  operands[1]));
 DONE;
 }
 operands[2] = GEN_INT (GET_MODE_MASK (<MODE>mode));
 }
 [(set_attr "type" "alu1")
 (set_attr "mode" "SI")])
 (define_insn "*zero_extend<mode>si2"
-[(set (match_operand:SI 0 "register_operand" "=r,*r")
+[(set (match_operand:SI 0 "register_operand" "=r,*r,*k")
 	(zero_extend:SI
-	  (match_operand:SWI12 1 "nonimmediate_operand" "<r>m,*k")))]
+	  (match_operand:SWI12 1 "nonimmediate_operand" "<r>m,*k,*km")))]
 "!(TARGET_ZERO_EXTEND_WITH_AND && optimize_function_for_speed_p (cfun))"
 "@
 movz{<imodesuffix>l|x}\t{%1, %0|%0, %1}
+kmov<mskmodesuffix>\t{%1, %0|%0, %1}
 kmov<mskmodesuffix>\t{%1, %0|%0, %1}"
-[(set_attr "isa" "*,<kmov_isa>")
+[(set_attr "isa" "*,<kmov_isa>,<kmov_isa>")
-(set_attr "type" "imovx,mskmov")
+(set_attr "type" "imovx,mskmov,mskmov")
-(set_attr "mode" "SI,<MODE>")])
+(set_attr "mode" "SI,<MODE>,<MODE>")])
 (define_expand "zero_extendqihi2"
 [(set (match_operand:HI 0 "register_operand")
 	(zero_extend:HI (match_operand:QI 1 "nonimmediate_operand")))]
 ""
 || REGNO (operands[0]) != REGNO (operands[1]))
 {
 ix86_expand_clear (operands[0]);
 gcc_assert (!TARGET_PARTIAL_REG_STALL);
-emit_insn (gen_movstrictqi
+emit_insn (gen_rtx_SET
-		  (gen_lowpart (QImode, operands[0]), operands[1]));
+		 (gen_rtx_STRICT_LOW_PART
+		  (VOIDmode, gen_lowpart (QImode, operands[0])),
+		  operands[1]));
 DONE;
 }
 operands[0] = gen_lowpart (SImode, operands[0]);
 }
 [(set_attr "type" "alu1")
 (set_attr "mode" "SI")])
 ; zero extend to SImode to avoid partial register stalls
 (define_insn "*zero_extendqihi2"
-[(set (match_operand:HI 0 "register_operand" "=r,*r")
+[(set (match_operand:HI 0 "register_operand" "=r,*r,*k")
-	(zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "qm,*k")))]
+	(zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "qm,*k,*km")))]
 "!(TARGET_ZERO_EXTEND_WITH_AND && optimize_function_for_speed_p (cfun))"
 "@
 movz{bl|x}\t{%1, %k0|%k0, %1}
-kmovb\t{%1, %k0|%k0, %1}"
+kmovb\t{%1, %k0|%k0, %1}
-[(set_attr "isa" "*,avx512dq")
+kmovb\t{%1, %0|%0, %1}"
-(set_attr "type" "imovx,mskmov")
+[(set_attr "isa" "*,avx512dq,avx512dq")
-(set_attr "mode" "SI,QI")])
+(set_attr "type" "imovx,mskmov,mskmov")
+(set_attr "mode" "SI,QI,QI")])
-(define_insn_and_split "*zext<mode>_doubleword_and"
-[(set (match_operand:DI 0 "register_operand" "=&<r>")
-	(zero_extend:DI (match_operand:SWI12 1 "nonimmediate_operand" "<r>m")))]
-"!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-&& TARGET_ZERO_EXTEND_WITH_AND && optimize_function_for_speed_p (cfun)"
-"#"
-"&& reload_completed && GENERAL_REG_P (operands[0])"
-[(set (match_dup 2) (const_int 0))]
-{
-split_double_mode (DImode, &operands[0], 1, &operands[0], &operands[2]);
-emit_move_insn (operands[0], const0_rtx);
-gcc_assert (!TARGET_PARTIAL_REG_STALL);
-emit_insn (gen_movstrict<mode>
-	     (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
-})
-(define_insn_and_split "*zext<mode>_doubleword"
-[(set (match_operand:DI 0 "register_operand" "=r")
-	(zero_extend:DI (match_operand:SWI12 1 "nonimmediate_operand" "<r>m")))]
-"!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-&& !(TARGET_ZERO_EXTEND_WITH_AND && optimize_function_for_speed_p (cfun))"
-"#"
-"&& reload_completed && GENERAL_REG_P (operands[0])"
-[(set (match_dup 0) (zero_extend:SI (match_dup 1)))
-(set (match_dup 2) (const_int 0))]
-"split_double_mode (DImode, &operands[0], 1, &operands[0], &operands[2]);")
-(define_insn_and_split "*zextsi_doubleword"
-[(set (match_operand:DI 0 "register_operand" "=r")
-	(zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "rm")))]
-"!TARGET_64BIT && TARGET_STV && TARGET_SSE2"
-"#"
-"&& reload_completed && GENERAL_REG_P (operands[0])"
-[(set (match_dup 0) (match_dup 1))
-(set (match_dup 2) (const_int 0))]
-"split_double_mode (DImode, &operands[0], 1, &operands[0], &operands[2]);")
 ;; Sign extension instructions
 (define_expand "extendsidi2"
 [(set (match_operand:DI 0 "register_operand")
 operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
 }
 })
 (define_insn "*extendsfdf2"
-[(set (match_operand:DF 0 "nonimm_ssenomem_operand" "=f,m,v")
+[(set (match_operand:DF 0 "nonimm_ssenomem_operand" "=f,m,v,v")
 (float_extend:DF
-	  (match_operand:SF 1 "nonimmediate_operand" "fm,f,vm")))]
+	  (match_operand:SF 1 "nonimmediate_operand" "fm,f,v,m")))]
 "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
 {
 switch (which_alternative)
 {
 case 0:
 case 1:
 return output_387_reg_move (insn, operands);
 case 2:
+return "%vcvtss2sd\t{%d1, %0|%0, %d1}";
+case 3:
 return "%vcvtss2sd\t{%1, %d0|%d0, %1}";
 default:
 gcc_unreachable ();
 }
 }
-[(set_attr "type" "fmov,fmov,ssecvt")
+[(set_attr "type" "fmov,fmov,ssecvt,ssecvt")
-(set_attr "prefix" "orig,orig,maybe_vex")
+(set_attr "avx_partial_xmm_update" "false,false,false,true")
-(set_attr "mode" "SF,XF,DF")
+(set_attr "prefix" "orig,orig,maybe_vex,maybe_vex")
+(set_attr "mode" "SF,XF,DF,DF")
 (set (attr "enabled")
 (if_then_else
 (match_test ("TARGET_SSE2 && TARGET_SSE_MATH"))
 (if_then_else
 	 (eq_attr "alternative" "0,1")
 && optimize_insn_for_speed_p ()"
 [(set (match_dup 2) (match_dup 1))
 (set (match_dup 0) (float_extend:DF (match_dup 2)))]
 "operands[2] = lowpart_subreg (SFmode, operands[0], DFmode);")
-;; Break partial reg stall for cvtss2sd.  This splitter should split
+;; Break partial SSE register dependency stall.  This splitter should split
-;; late in the pass sequence (after register rename pass),
+;; late in the pass sequence (after register rename pass), so allocated
-;; so allocated registers won't change anymore.
+;; registers won't change anymore
 (define_split
 [(set (match_operand:DF 0 "sse_reg_operand")
 (float_extend:DF
 (match_operand:SF 1 "nonimmediate_operand")))]
-"TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
+"!TARGET_AVX
+&& TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
 && optimize_function_for_speed_p (cfun)
 && (!REG_P (operands[1])
-|| REGNO (operands[0]) != REGNO (operands[1]))
+|| (!TARGET_AVX && REGNO (operands[0]) != REGNO (operands[1])))
 && (!EXT_REX_SSE_REG_P (operands[0])
 || TARGET_AVX512VL)"
 [(set (match_dup 0)
 (vec_merge:V2DF
 	  (vec_duplicate:V2DF
 ;; register preferencing, but we really use a stack slot.
 ;; Conversion from DFmode to SFmode.
 (define_insn "truncdfsf2"
-[(set (match_operand:SF 0 "nonimm_ssenomem_operand" "=m,f,v")
+[(set (match_operand:SF 0 "nonimm_ssenomem_operand" "=m,f,v,v")
 	(float_truncate:SF
-	  (match_operand:DF 1 "register_ssemem_operand" "f,f,vm")))]
+	  (match_operand:DF 1 "register_ssemem_operand" "f,f,v,m")))]
 "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
 {
 switch (which_alternative)
 {
 case 0:
 case 1:
 return output_387_reg_move (insn, operands);
 case 2:
+return "%vcvtsd2ss\t{%d1, %0|%0, %d1}";
+case 3:
 return "%vcvtsd2ss\t{%1, %d0|%d0, %1}";
 default:
 gcc_unreachable ();
 }
 }
-[(set_attr "type" "fmov,fmov,ssecvt")
+[(set_attr "type" "fmov,fmov,ssecvt,ssecvt")
+(set_attr "avx_partial_xmm_update" "false,false,false,true")
 (set_attr "mode" "SF")
 (set (attr "enabled")
 (if_then_else
 (match_test ("TARGET_SSE2 && TARGET_SSE_MATH"))
 (cond [(eq_attr "alternative" "0")
 && optimize_insn_for_speed_p ()"
 [(set (match_dup 2) (match_dup 1))
 (set (match_dup 0) (float_truncate:SF (match_dup 2)))]
 "operands[2] = lowpart_subreg (DFmode, operands[0], SFmode);")
-;; Break partial reg stall for cvtsd2ss.  This splitter should split
+;; Break partial SSE register dependency stall.  This splitter should split
-;; late in the pass sequence (after register rename pass),
+;; late in the pass sequence (after register rename pass), so allocated
-;; so allocated registers won't change anymore.
+;; registers won't change anymore
 (define_split
 [(set (match_operand:SF 0 "sse_reg_operand")
 (float_truncate:SF
 	  (match_operand:DF 1 "nonimmediate_operand")))]
-"TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
+"!TARGET_AVX
+&& TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
 && optimize_function_for_speed_p (cfun)
 && (!REG_P (operands[1])
-|| REGNO (operands[0]) != REGNO (operands[1]))
+|| (!TARGET_AVX && REGNO (operands[0]) != REGNO (operands[1])))
 && (!EXT_REX_SSE_REG_P (operands[0])
 || TARGET_AVX512VL)"
 [(set (match_dup 0)
 	(vec_merge:V4SF
 	  (vec_duplicate:V4SF
 (clobber (reg:CC FLAGS_REG))]
 "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
 && !TARGET_FISTTP
 && !(SSE_FLOAT_MODE_P (GET_MODE (operands[1]))
 	 && (TARGET_64BIT || <MODE>mode != DImode))
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 ix86_optimize_mode_switching[I387_TRUNC] = 1;
 "@
 fild%Z1\t%1
 %vcvtsi2<MODEF:ssemodesuffix><SWI48:rex64suffix>\t{%1, %d0|%d0, %1}
 %vcvtsi2<MODEF:ssemodesuffix><SWI48:rex64suffix>\t{%1, %d0|%d0, %1}"
 [(set_attr "type" "fmov,sseicvt,sseicvt")
+(set_attr "avx_partial_xmm_update" "false,true,true")
 (set_attr "prefix" "orig,maybe_vex,maybe_vex")
 (set_attr "mode" "<MODEF:MODE>")
 (set (attr "prefix_rex")
 (if_then_else
 (and (eq_attr "prefix" "maybe_vex")
 	      assign_386_stack_local (DImode, SLOT_TEMP)));
 DONE;
 })
 (define_insn_and_split "floatdi<X87MODEF:mode>2_i387_with_xmm"
-[(set (match_operand:X87MODEF 0 "register_operand" "=f")
+[(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
 	(float:X87MODEF
-	  (match_operand:DI 1 "register_operand" "r")))
+	  (match_operand:DI 1 "register_operand" "r,r")))
-(clobber (match_scratch:V4SI 3 "=x"))
+(clobber (match_operand:DI 2 "memory_operand" "=m,m"))
-(clobber (match_scratch:V4SI 4 "=x"))
+(clobber (match_scratch:V4SI 3 "=x,x"))
-(clobber (match_operand:DI 2 "memory_operand" "=m"))]
+(clobber (match_scratch:V4SI 4 "=X,x"))]
 "!TARGET_64BIT && TARGET_INTER_UNIT_MOVES_TO_VEC
 && TARGET_80387 && X87_ENABLE_FLOAT (<X87MODEF:MODE>mode, DImode)
 && TARGET_SSE2 && optimize_function_for_speed_p (cfun)"
 "#"
 "&& reload_completed"
 {
 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
 Assemble the 64-bit DImode value in an xmm register.  */
 emit_insn (gen_sse2_loadld (operands[3], CONST0_RTX (V4SImode),
 			      gen_lowpart (SImode, operands[1])));
-emit_insn (gen_sse2_loadld (operands[4], CONST0_RTX (V4SImode),
+if (TARGET_SSE4_1)
-			      gen_highpart (SImode, operands[1])));
+emit_insn (gen_sse4_1_pinsrd (operands[3], operands[3],
-emit_insn (gen_vec_interleave_lowv4si (operands[3], operands[3],
+				  gen_highpart (SImode, operands[1]),
-					 operands[4]));
+				  GEN_INT (2)));
+else
+{
+emit_insn (gen_sse2_loadld (operands[4], CONST0_RTX (V4SImode),
+				  gen_highpart (SImode, operands[1])));
+emit_insn (gen_vec_interleave_lowv4si (operands[3], operands[3],
+					     operands[4]));
+}
 operands[3] = gen_lowpart (DImode, operands[3]);
 }
-[(set_attr "type" "multi")
+[(set_attr "isa" "sse4,*")
+(set_attr "type" "multi")
 (set_attr "mode" "<X87MODEF:MODE>")
 (set_attr "unit" "i387")
 (set_attr "fp_int_src" "true")])
-;; Avoid partial SSE register dependency stalls.  This splitter should split
+;; Break partial SSE register dependency stall.  This splitter should split
 ;; late in the pass sequence (after register rename pass), so allocated
 ;; registers won't change anymore
 (define_split
 [(set (match_operand:MODEF 0 "sse_reg_operand")
 	(float:MODEF (match_operand:SWI48 1 "nonimmediate_operand")))]
-"TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
+"!TARGET_AVX
+&& TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
 && optimize_function_for_speed_p (cfun)
 && (!EXT_REX_SSE_REG_P (operands[0])
 || TARGET_AVX512VL)"
 [(set (match_dup 0)
 	(vec_merge:<MODEF:ssevecmode>
 	(unsigned_float:MODEF
 	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")))]
 "TARGET_AVX512F && TARGET_SSE_MATH"
 "vcvtusi2<MODEF:ssemodesuffix><SWI48:rex64suffix>\t{%1, %0, %0|%0, %0, %1}"
 [(set_attr "type" "sseicvt")
+(set_attr "avx_partial_xmm_update" "true")
 (set_attr "prefix" "evex")
 (set_attr "mode" "<MODEF:MODE>")])
 ;; Avoid store forwarding (partial memory) stall penalty by extending
 ;; SImode value to DImode through XMM register instead of pushing two
 (define_expand "floatunsdidf2"
 [(set (match_operand:DF 0 "register_operand")
 	(unsigned_float:DF
 	  (match_operand:DI 1 "nonimmediate_operand")))]
-"(TARGET_KEEPS_VECTOR_ALIGNED_STACK || TARGET_AVX512F)
+"((TARGET_64BIT && TARGET_AVX512F)
+|| TARGET_KEEPS_VECTOR_ALIGNED_STACK)
 && TARGET_SSE2 && TARGET_SSE_MATH"
 {
 if (!TARGET_64BIT)
 {
 ix86_expand_convert_uns_didf_sse (operands[0], operands[1]);
 		    (match_operand:SDWIM 2 "<general_hilo_operand>")))]
 ""
 "ix86_expand_binary_operator (PLUS, <MODE>mode, operands); DONE;")
 (define_insn_and_split "*add<dwi>3_doubleword"
-[(set (match_operand:<DWI> 0 "nonimmediate_operand" "=r,o")
+[(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
 	(plus:<DWI>
 	  (match_operand:<DWI> 1 "nonimmediate_operand" "%0,0")
-	  (match_operand:<DWI> 2 "x86_64_hilo_general_operand"
+	  (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "r<di>,o")))
-							"ro<di>,r<di>")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (PLUS, <DWI>mode, operands)"
 "#"
 "reload_completed"
 [(parallel [(set (reg:CCC FLAGS_REG)
 DONE;
 }
 })
 (define_insn "*add<mode>_1"
-[(set (match_operand:SWI48 0 "nonimmediate_operand" "=r,rm,r,r")
+[(set (match_operand:SWI48 0 "nonimmediate_operand" "=rm,r,r,r")
 	(plus:SWI48
 	  (match_operand:SWI48 1 "nonimmediate_operand" "%0,0,r,r")
-	  (match_operand:SWI48 2 "x86_64_general_operand" "rme,re,0,le")))
+	  (match_operand:SWI48 2 "x86_64_general_operand" "re,m,0,le")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 {
 switch (get_attr_type (insn))
 {
 (set_attr "mode" "SI")])
 (define_insn "*addhi_1"
 [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r,r,Yp")
 	(plus:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,r,Yp")
-		 (match_operand:HI 2 "general_operand" "rn,rm,0,ln")))
+		 (match_operand:HI 2 "general_operand" "rn,m,0,ln")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (PLUS, HImode, operands)"
 {
 switch (get_attr_type (insn))
 {
 (set_attr "mode" "HI,HI,HI,SI")])
 (define_insn "*addqi_1"
 [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,q,r,r,Yp")
 	(plus:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,q,0,r,Yp")
-		 (match_operand:QI 2 "general_operand" "qn,qm,0,rn,0,ln")))
+		 (match_operand:QI 2 "general_operand" "qn,m,0,rn,0,ln")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (PLUS, QImode, operands)"
 {
 bool widen = (get_attr_mode (insn) != MODE_QI);
 (set (attr "preferred_for_speed")
 (cond [(eq_attr "alternative" "3,4")
 	      (symbol_ref "!TARGET_PARTIAL_REG_STALL")]
 	   (symbol_ref "true")))])
-(define_insn "*addqi_1_slp"
+(define_insn "*add<mode>_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(plus:QI (match_dup 0)
+	(plus:SWI12 (match_operand:SWI12 1 "nonimmediate_operand" "%0")
-		 (match_operand:QI 1 "general_operand" "qn,qm")))
+		    (match_operand:SWI12 2 "general_operand" "<r>mn")))
 (clobber (reg:CC FLAGS_REG))]
-"(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
+&& (rtx_equal_p (operands[0], operands[1])
+|| rtx_equal_p (operands[0], operands[2]))"
 {
 switch (get_attr_type (insn))
 {
 case TYPE_INCDEC:
-if (operands[1] == const1_rtx)
+if (operands[2] == const1_rtx)
-	return "inc{b}\t%0";
+	return "inc{<imodesuffix>}\t%0";
 else
 	{
-	  gcc_assert (operands[1] == constm1_rtx);
+	  gcc_assert (operands[2] == constm1_rtx);
-	  return "dec{b}\t%0";
+	  return "dec{<imodesuffix>}\t%0";
 	}
 default:
-if (x86_maybe_negate_const_int (&operands[1], QImode))
+if (x86_maybe_negate_const_int (&operands[2], QImode))
-	return "sub{b}\t{%1, %0|%0, %1}";
+	return "sub{<imodesuffix>}\t{%2, %0|%0, %2}";
-return "add{b}\t{%1, %0|%0, %1}";
+return "add{<imodesuffix>}\t{%2, %0|%0, %2}";
 }
 }
 [(set (attr "type")
-(if_then_else (match_operand:QI 1 "incdec_operand")
+(if_then_else (match_operand:QI 2 "incdec_operand")
 	(const_string "incdec")
-	(const_string "alu1")))
+	(const_string "alu")))
-(set (attr "memory")
+(set_attr "mode" "<MODE>")])
-(if_then_else (match_operand 1 "memory_operand")
-(const_string "load")
-(const_string "none")))
-(set_attr "mode" "QI")])
 ;; Split non destructive adds if we cannot use lea.
 (define_split
 [(set (match_operand:SWI48 0 "register_operand")
 	(plus:SWI48 (match_operand:SWI48 1 "register_operand")
 (define_insn "*add<mode>_2"
 [(set (reg FLAGS_REG)
 	(compare
 	  (plus:SWI
 	    (match_operand:SWI 1 "nonimmediate_operand" "%0,0,<r>")
-	    (match_operand:SWI 2 "<general_operand>" "<g>,<r><i>,0"))
+	    (match_operand:SWI 2 "<general_operand>" "<r><i>,m,0"))
 	  (const_int 0)))
-(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m,<r>")
+(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>,<r>")
 	(plus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_match_ccmode (insn, CCGOCmode)
 && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 {
 switch (get_attr_type (insn))
 || rtx_equal_p (operands[0], operands[2])"
 "add{b}\t{%h2, %h0|%h0, %h2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "QI")])
+;; Like DWI, but use POImode instead of OImode.
+(define_mode_attr DPWI [(QI "HI") (HI "SI") (SI "DI") (DI "TI") (TI "POI")])
 ;; Add with jump on overflow.
 (define_expand "addv<mode>4"
 [(parallel [(set (reg:CCO FLAGS_REG)
-		   (eq:CCO (plus:<DWI>
+		   (eq:CCO
-			      (sign_extend:<DWI>
+		     (plus:<DPWI>
-				 (match_operand:SWI 1 "nonimmediate_operand"))
+		       (sign_extend:<DPWI>
-			      (match_dup 4))
+			 (match_operand:SWIDWI 1 "nonimmediate_operand"))
-			   (sign_extend:<DWI>
+		       (match_dup 4))
-			      (plus:SWI (match_dup 1)
+			 (sign_extend:<DPWI>
-					(match_operand:SWI 2
+			   (plus:SWIDWI (match_dup 1)
-					   "<general_operand>")))))
+			     (match_operand:SWIDWI 2
-	      (set (match_operand:SWI 0 "register_operand")
+			       "<general_hilo_operand>")))))
-		   (plus:SWI (match_dup 1) (match_dup 2)))])
+	      (set (match_operand:SWIDWI 0 "register_operand")
+		   (plus:SWIDWI (match_dup 1) (match_dup 2)))])
 (set (pc) (if_then_else
 	       (eq (reg:CCO FLAGS_REG) (const_int 0))
 	       (label_ref (match_operand 3))
 	       (pc)))]
 ""
 {
 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
-if (CONST_INT_P (operands[2]))
+if (CONST_SCALAR_INT_P (operands[2]))
 operands[4] = operands[2];
 else
-operands[4] = gen_rtx_SIGN_EXTEND (<DWI>mode, operands[2]);
+operands[4] = gen_rtx_SIGN_EXTEND (<DPWI>mode, operands[2]);
 })
 (define_insn "*addv<mode>4"
 [(set (reg:CCO FLAGS_REG)
 	(eq:CCO (plus:<DWI>
 		   (sign_extend:<DWI>
 		      (match_operand:SWI 1 "nonimmediate_operand" "%0,0"))
 		   (sign_extend:<DWI>
-		      (match_operand:SWI 2 "<general_sext_operand>"
+		      (match_operand:SWI 2 "<general_sext_operand>" "<r>We,m")))
-					   "<r>mWe,<r>We")))
 		(sign_extend:<DWI>
 		   (plus:SWI (match_dup 1) (match_dup 2)))))
-(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m")
+(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(plus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 "add{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
-(define_insn "*addv<mode>4_1"
+(define_insn "addv<mode>4_1"
 [(set (reg:CCO FLAGS_REG)
 	(eq:CCO (plus:<DWI>
 		   (sign_extend:<DWI>
 		      (match_operand:SWI 1 "nonimmediate_operand" "0"))
 		   (match_operand:<DWI> 3 "const_int_operand" "i"))
 		(sign_extend:<DWI>
-		   (plus:SWI (match_dup 1)
+		   (plus:SWI
-			     (match_operand:SWI 2 "x86_64_immediate_operand"
+		     (match_dup 1)
-						  "<i>")))))
+		     (match_operand:SWI 2 "x86_64_immediate_operand" "<i>")))))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m")
 	(plus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
 && CONST_INT_P (operands[2])
 && INTVAL (operands[2]) == INTVAL (operands[3])"
 		  (const_string "1")
 	       (match_test "<MODE_SIZE> == 8")
 		  (const_string "4")]
 	      (const_string "<MODE_SIZE>")))])
+;; Quad word integer modes as mode attribute.
+(define_mode_attr QPWI [(SI "TI") (DI "POI")])
+(define_insn_and_split "*addv<dwi>4_doubleword"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (plus:<QPWI>
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 1 "nonimmediate_operand" "%0,0"))
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 2 "nonimmediate_operand" "r,o")))
+	  (sign_extend:<QPWI>
+	    (plus:<DWI> (match_dup 1) (match_dup 2)))))
+(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
+	(plus:<DWI> (match_dup 1) (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <DWI>mode, operands)"
+"#"
+"reload_completed"
+[(parallel [(set (reg:CCC FLAGS_REG)
+		   (compare:CCC
+		     (plus:DWIH (match_dup 1) (match_dup 2))
+		     (match_dup 1)))
+	      (set (match_dup 0)
+		   (plus:DWIH (match_dup 1) (match_dup 2)))])
+(parallel [(set (reg:CCO FLAGS_REG)
+		   (eq:CCO
+		     (plus:<DWI>
+		       (plus:<DWI>
+			 (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0))
+			 (sign_extend:<DWI> (match_dup 4)))
+		       (sign_extend:<DWI> (match_dup 5)))
+		     (sign_extend:<DWI>
+		       (plus:DWIH
+			 (plus:DWIH
+			   (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+			   (match_dup 4))
+			 (match_dup 5)))))
+	      (set (match_dup 3)
+		   (plus:DWIH
+		     (plus:DWIH
+		       (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+		       (match_dup 4))
+		     (match_dup 5)))])]
+{
+split_double_mode (<DWI>mode, &operands[0], 3, &operands[0], &operands[3]);
+})
+(define_insn_and_split "*addv<dwi>4_doubleword_1"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (plus:<QPWI>
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 1 "nonimmediate_operand" "%0"))
+	    (match_operand:<QPWI> 3 "const_scalar_int_operand" ""))
+	  (sign_extend:<QPWI>
+	    (plus:<DWI>
+	      (match_dup 1)
+	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "<di>")))))
+(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro")
+	(plus:<DWI> (match_dup 1) (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <DWI>mode, operands)
+&& CONST_SCALAR_INT_P (operands[2])
+&& rtx_equal_p (operands[2], operands[3])"
+"#"
+"reload_completed"
+[(parallel [(set (reg:CCC FLAGS_REG)
+		   (compare:CCC
+		     (plus:DWIH (match_dup 1) (match_dup 2))
+		     (match_dup 1)))
+	      (set (match_dup 0)
+		   (plus:DWIH (match_dup 1) (match_dup 2)))])
+(parallel [(set (reg:CCO FLAGS_REG)
+		   (eq:CCO
+		     (plus:<DWI>
+		       (plus:<DWI>
+			 (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0))
+			 (sign_extend:<DWI> (match_dup 4)))
+		       (match_dup 5))
+		     (sign_extend:<DWI>
+		       (plus:DWIH
+			 (plus:DWIH
+			   (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+			   (match_dup 4))
+			 (match_dup 5)))))
+	      (set (match_dup 3)
+		   (plus:DWIH
+		     (plus:DWIH
+		       (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+		       (match_dup 4))
+		     (match_dup 5)))])]
+{
+split_double_mode (<DWI>mode, &operands[0], 3, &operands[0], &operands[3]);
+if (operands[2] == const0_rtx)
+{
+emit_insn (gen_addv<mode>4_1 (operands[3], operands[4], operands[5],
+				    operands[5]));
+DONE;
+}
+})
+(define_insn "*addv<mode>4_overflow_1"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (plus:<DWI>
+	    (plus:<DWI>
+	      (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+		[(match_operand 3 "flags_reg_operand") (const_int 0)])
+	      (sign_extend:<DWI>
+		(match_operand:SWI 1 "nonimmediate_operand" "%0,0")))
+	    (sign_extend:<DWI>
+	      (match_operand:SWI 2 "<general_sext_operand>" "rWe,m")))
+	  (sign_extend:<DWI>
+	    (plus:SWI
+	      (plus:SWI
+		(match_operator:SWI 5 "ix86_carry_flag_operator"
+		  [(match_dup 3) (const_int 0)])
+		(match_dup 1))
+	      (match_dup 2)))))
+(set (match_operand:SWI 0 "nonimmediate_operand" "=rm,r")
+	(plus:SWI
+	  (plus:SWI
+	    (match_op_dup 5 [(match_dup 3) (const_int 0)])
+	    (match_dup 1))
+	  (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
+"adc{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")])
+(define_insn "*addv<mode>4_overflow_2"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (plus:<DWI>
+	    (plus:<DWI>
+	      (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+		[(match_operand 3 "flags_reg_operand") (const_int 0)])
+	      (sign_extend:<DWI>
+		(match_operand:SWI 1 "nonimmediate_operand" "%0")))
+	    (match_operand:<DWI> 6 "const_int_operand" ""))
+	  (sign_extend:<DWI>
+	    (plus:SWI
+	      (plus:SWI
+		(match_operator:SWI 5 "ix86_carry_flag_operator"
+		  [(match_dup 3) (const_int 0)])
+		(match_dup 1))
+	      (match_operand:SWI 2 "x86_64_immediate_operand" "e")))))
+(set (match_operand:SWI 0 "nonimmediate_operand" "=rm")
+	(plus:SWI
+	  (plus:SWI
+	    (match_op_dup 5 [(match_dup 3) (const_int 0)])
+	    (match_dup 1))
+	  (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
+&& CONST_INT_P (operands[2])
+&& INTVAL (operands[2]) == INTVAL (operands[6])"
+"adc{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")
+(set (attr "length_immediate")
+(if_then_else (match_test "IN_RANGE (INTVAL (operands[2]), -128, 127)")
+(const_string "1")
+(const_string "4")))])
 (define_expand "uaddv<mode>4"
 [(parallel [(set (reg:CCC FLAGS_REG)
 		   (compare:CCC
-		     (plus:SWI
+		     (plus:SWIDWI
-		       (match_operand:SWI 1 "nonimmediate_operand")
+		       (match_operand:SWIDWI 1 "nonimmediate_operand")
-		       (match_operand:SWI 2 "<general_operand>"))
+		       (match_operand:SWIDWI 2 "<general_hilo_operand>"))
 		     (match_dup 1)))
-	      (set (match_operand:SWI 0 "register_operand")
+	      (set (match_operand:SWIDWI 0 "register_operand")
-		   (plus:SWI (match_dup 1) (match_dup 2)))])
+		   (plus:SWIDWI (match_dup 1) (match_dup 2)))])
 (set (pc) (if_then_else
 	       (ltu (reg:CCC FLAGS_REG) (const_int 0))
 	       (label_ref (match_operand 3))
 	       (pc)))]
 ""
 		     (match_operand:SDWIM 2 "<general_hilo_operand>")))]
 ""
 "ix86_expand_binary_operator (MINUS, <MODE>mode, operands); DONE;")
 (define_insn_and_split "*sub<dwi>3_doubleword"
-[(set (match_operand:<DWI> 0 "nonimmediate_operand" "=r,o")
+[(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
 	(minus:<DWI>
 	  (match_operand:<DWI> 1 "nonimmediate_operand" "0,0")
-	  (match_operand:<DWI> 2 "x86_64_hilo_general_operand"
+	  (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "r<di>,o")))
-							"ro<di>,r<di>")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 "#"
 "reload_completed"
 [(parallel [(set (reg:CC FLAGS_REG)
 (define_insn "*sub<mode>_1"
 [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI
 	  (match_operand:SWI 1 "nonimmediate_operand" "0,0")
-	  (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
+	  (match_operand:SWI 2 "<general_operand>" "<r><i>,m")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
 "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
 "sub{l}\t{%2, %k0|%k0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "SI")])
-(define_insn "*subqi_1_slp"
+(define_insn "*sub<mode>_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(minus:QI (match_dup 0)
+	(minus:SWI12 (match_operand:SWI12 1 "register_operand" "0")
-		  (match_operand:QI 1 "general_operand" "qn,qm")))
+		     (match_operand:SWI12 2 "general_operand" "<r>mn")))
 (clobber (reg:CC FLAGS_REG))]
-"(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-"sub{b}\t{%1, %0|%0, %1}"
+&& rtx_equal_p (operands[0], operands[1])"
-[(set_attr "type" "alu1")
+"sub{<imodesuffix>}\t{%2, %0|%0, %2}"
-(set_attr "mode" "QI")])
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")])
 (define_insn "*sub<mode>_2"
 [(set (reg FLAGS_REG)
 	(compare
 	  (minus:SWI
 	    (match_operand:SWI 1 "nonimmediate_operand" "0,0")
-	    (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))
+	    (match_operand:SWI 2 "<general_operand>" "<r><i>,m"))
 	  (const_int 0)))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_match_ccmode (insn, CCGOCmode)
 && ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 (set_attr "mode" "SI")])
 ;; Subtract with jump on overflow.
 (define_expand "subv<mode>4"
 [(parallel [(set (reg:CCO FLAGS_REG)
-		   (eq:CCO (minus:<DWI>
+		   (eq:CCO
-			      (sign_extend:<DWI>
+		     (minus:<DPWI>
-				 (match_operand:SWI 1 "nonimmediate_operand"))
+		       (sign_extend:<DPWI>
-			      (match_dup 4))
+			 (match_operand:SWIDWI 1 "nonimmediate_operand"))
-			   (sign_extend:<DWI>
+		       (match_dup 4))
-			      (minus:SWI (match_dup 1)
+		     (sign_extend:<DPWI>
-					 (match_operand:SWI 2
+		       (minus:SWIDWI (match_dup 1)
-					    "<general_operand>")))))
+				     (match_operand:SWIDWI 2
-	      (set (match_operand:SWI 0 "register_operand")
+						"<general_hilo_operand>")))))
-		   (minus:SWI (match_dup 1) (match_dup 2)))])
+	      (set (match_operand:SWIDWI 0 "register_operand")
+		   (minus:SWIDWI (match_dup 1) (match_dup 2)))])
 (set (pc) (if_then_else
 	       (eq (reg:CCO FLAGS_REG) (const_int 0))
 	       (label_ref (match_operand 3))
 	       (pc)))]
 ""
 {
 ix86_fixup_binary_operands_no_copy (MINUS, <MODE>mode, operands);
-if (CONST_INT_P (operands[2]))
+if (CONST_SCALAR_INT_P (operands[2]))
 operands[4] = operands[2];
 else
-operands[4] = gen_rtx_SIGN_EXTEND (<DWI>mode, operands[2]);
+operands[4] = gen_rtx_SIGN_EXTEND (<DPWI>mode, operands[2]);
 })
 (define_insn "*subv<mode>4"
 [(set (reg:CCO FLAGS_REG)
 	(eq:CCO (minus:<DWI>
 		   (sign_extend:<DWI>
 		      (match_operand:SWI 1 "nonimmediate_operand" "0,0"))
 		   (sign_extend:<DWI>
-		      (match_operand:SWI 2 "<general_sext_operand>"
+		      (match_operand:SWI 2 "<general_sext_operand>" "<r>We,m")))
-					   "<r>We,<r>m")))
 		(sign_extend:<DWI>
 		   (minus:SWI (match_dup 1) (match_dup 2)))))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
-(define_insn "*subv<mode>4_1"
+(define_insn "subv<mode>4_1"
 [(set (reg:CCO FLAGS_REG)
 	(eq:CCO (minus:<DWI>
 		   (sign_extend:<DWI>
 		      (match_operand:SWI 1 "nonimmediate_operand" "0"))
 		   (match_operand:<DWI> 3 "const_int_operand" "i"))
 		(sign_extend:<DWI>
-		   (minus:SWI (match_dup 1)
+		   (minus:SWI
-			      (match_operand:SWI 2 "x86_64_immediate_operand"
+		     (match_dup 1)
-						   "<i>")))))
+		     (match_operand:SWI 2 "x86_64_immediate_operand" "<i>")))))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m")
 	(minus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
 && CONST_INT_P (operands[2])
 && INTVAL (operands[2]) == INTVAL (operands[3])"
 		  (const_string "1")
 	       (match_test "<MODE_SIZE> == 8")
 		  (const_string "4")]
 	      (const_string "<MODE_SIZE>")))])
+(define_insn_and_split "*subv<dwi>4_doubleword"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (minus:<QPWI>
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 1 "nonimmediate_operand" "0,0"))
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 2 "nonimmediate_operand" "r,o")))
+	  (sign_extend:<QPWI>
+	    (minus:<DWI> (match_dup 1) (match_dup 2)))))
+(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
+	(minus:<DWI> (match_dup 1) (match_dup 2)))]
+"ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
+"#"
+"reload_completed"
+[(parallel [(set (reg:CC FLAGS_REG)
+		   (compare:CC (match_dup 1) (match_dup 2)))
+	      (set (match_dup 0)
+		   (minus:DWIH (match_dup 1) (match_dup 2)))])
+(parallel [(set (reg:CCO FLAGS_REG)
+		   (eq:CCO
+		     (minus:<DWI>
+		       (minus:<DWI>
+			 (sign_extend:<DWI> (match_dup 4))
+			 (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0)))
+		       (sign_extend:<DWI> (match_dup 5)))
+		     (sign_extend:<DWI>
+		       (minus:DWIH
+			 (minus:DWIH
+			   (match_dup 4)
+			   (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0)))
+			 (match_dup 5)))))
+	      (set (match_dup 3)
+		   (minus:DWIH
+		     (minus:DWIH
+		       (match_dup 4)
+		       (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0)))
+		     (match_dup 5)))])]
+{
+split_double_mode (<DWI>mode, &operands[0], 3, &operands[0], &operands[3]);
+})
+(define_insn_and_split "*subv<dwi>4_doubleword_1"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (minus:<QPWI>
+	    (sign_extend:<QPWI>
+	      (match_operand:<DWI> 1 "nonimmediate_operand" "0"))
+	    (match_operand:<QPWI> 3 "const_scalar_int_operand" ""))
+	  (sign_extend:<QPWI>
+	    (minus:<DWI>
+	      (match_dup 1)
+	      (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "<di>")))))
+(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro")
+	(minus:<DWI> (match_dup 1) (match_dup 2)))]
+"ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
+&& CONST_SCALAR_INT_P (operands[2])
+&& rtx_equal_p (operands[2], operands[3])"
+"#"
+"reload_completed"
+[(parallel [(set (reg:CC FLAGS_REG)
+		   (compare:CC (match_dup 1) (match_dup 2)))
+	      (set (match_dup 0)
+		   (minus:DWIH (match_dup 1) (match_dup 2)))])
+(parallel [(set (reg:CCO FLAGS_REG)
+		   (eq:CCO
+		     (minus:<DWI>
+		       (minus:<DWI>
+			 (sign_extend:<DWI> (match_dup 4))
+			 (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0)))
+		       (match_dup 5))
+		     (sign_extend:<DWI>
+		       (minus:DWIH
+			 (minus:DWIH
+			   (match_dup 4)
+			   (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0)))
+			 (match_dup 5)))))
+	      (set (match_dup 3)
+		   (minus:DWIH
+		     (minus:DWIH
+		       (match_dup 4)
+		       (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0)))
+		     (match_dup 5)))])]
+{
+split_double_mode (<DWI>mode, &operands[0], 3, &operands[0], &operands[3]);
+if (operands[2] == const0_rtx)
+{
+emit_insn (gen_subv<mode>4_1 (operands[3], operands[4], operands[5],
+				    operands[5]));
+DONE;
+}
+})
+(define_insn "*subv<mode>4_overflow_1"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (minus:<DWI>
+	    (minus:<DWI>
+	      (sign_extend:<DWI>
+		(match_operand:SWI 1 "nonimmediate_operand" "%0,0"))
+	      (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+		[(match_operand 3 "flags_reg_operand") (const_int 0)]))
+	    (sign_extend:<DWI>
+	      (match_operand:SWI 2 "<general_sext_operand>" "rWe,m")))
+	  (sign_extend:<DWI>
+	    (minus:SWI
+	      (minus:SWI
+		(match_dup 1)
+		(match_operator:SWI 5 "ix86_carry_flag_operator"
+		  [(match_dup 3) (const_int 0)]))
+	      (match_dup 2)))))
+(set (match_operand:SWI 0 "nonimmediate_operand" "=rm,r")
+	(minus:SWI
+	  (minus:SWI
+	    (match_dup 1)
+	    (match_op_dup 5 [(match_dup 3) (const_int 0)]))
+	  (match_dup 2)))]
+"ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
+"sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")])
+(define_insn "*subv<mode>4_overflow_2"
+[(set (reg:CCO FLAGS_REG)
+	(eq:CCO
+	  (minus:<DWI>
+	    (minus:<DWI>
+	      (sign_extend:<DWI>
+		(match_operand:SWI 1 "nonimmediate_operand" "%0"))
+	      (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+		[(match_operand 3 "flags_reg_operand") (const_int 0)]))
+	    (match_operand:<DWI> 6 "const_int_operand" ""))
+	  (sign_extend:<DWI>
+	    (minus:SWI
+	      (minus:SWI
+		(match_dup 1)
+		(match_operator:SWI 5 "ix86_carry_flag_operator"
+		  [(match_dup 3) (const_int 0)]))
+	      (match_operand:SWI 2 "x86_64_immediate_operand" "e")))))
+(set (match_operand:SWI 0 "nonimmediate_operand" "=rm")
+	(minus:SWI
+	  (minus:SWI
+	    (match_dup 1)
+	    (match_op_dup 5 [(match_dup 3) (const_int 0)]))
+	  (match_dup 2)))]
+"ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
+&& CONST_INT_P (operands[2])
+&& INTVAL (operands[2]) == INTVAL (operands[6])"
+"sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")
+(set (attr "length_immediate")
+(if_then_else (match_test "IN_RANGE (INTVAL (operands[2]), -128, 127)")
+(const_string "1")
+(const_string "4")))])
 (define_expand "usubv<mode>4"
 [(parallel [(set (reg:CC FLAGS_REG)
 		   (compare:CC
 		     (match_operand:SWI 1 "nonimmediate_operand")
 		     (match_operand:SWI 2 "<general_operand>")))
 "ix86_fixup_binary_operands_no_copy (MINUS, <MODE>mode, operands);")
 (define_insn "*sub<mode>_3"
 [(set (reg FLAGS_REG)
 	(compare (match_operand:SWI 1 "nonimmediate_operand" "0,0")
-		 (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
+		 (match_operand:SWI 2 "<general_operand>" "<r><i>,m")))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_match_ccmode (insn, CCmode)
 && ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
 (set (match_dup 0)
 	   (minus:SWI (match_dup 0) (match_dup 1)))])]
 "find_regno_note (peep2_next_insn (0), REG_UNUSED, REGNO (operands[0])) != 0"
 [(set (reg:CC FLAGS_REG)
 	(compare:CC (match_dup 0) (match_dup 1)))])
+;; decl %eax; cmpl $-1, %eax; jne .Lxx; can be optimized into
+;; subl $1, %eax; jnc .Lxx;
+(define_peephole2
+[(parallel
+[(set (match_operand:SWI 0 "general_reg_operand")
+	   (plus:SWI (match_dup 0) (const_int -1)))
+(clobber (reg FLAGS_REG))])
+(set (reg:CCZ FLAGS_REG)
+	(compare:CCZ (match_dup 0) (const_int -1)))
+(set (pc)
+	(if_then_else (match_operator 1 "bt_comparison_operator"
+			[(reg:CCZ FLAGS_REG) (const_int 0)])
+		      (match_operand 2)
+		      (pc)))]
+"peep2_regno_dead_p (3, FLAGS_REG)"
+[(parallel
+[(set (reg:CC FLAGS_REG)
+	    (compare:CC (match_dup 0) (const_int 1)))
+(set (match_dup 0)
+	    (minus:SWI (match_dup 0) (const_int 1)))])
+(set (pc)
+	 (if_then_else (match_dup 3)
+		       (match_dup 2)
+		       (pc)))]
+{
+rtx cc = gen_rtx_REG (CCmode, FLAGS_REG);
+operands[3] = gen_rtx_fmt_ee (GET_CODE (operands[1]) == NE
+				? GEU : LTU, VOIDmode, cc, const0_rtx);
+})
 (define_insn "*subsi_3_zext"
 [(set (reg FLAGS_REG)
 	(compare (match_operand:SI 1 "register_operand" "0")
 		 (match_operand:SI 2 "x86_64_general_operand" "rme")))
 [(set_attr "type" "alu")
 (set_attr "mode" "SI")])
 ;; Add with carry and subtract with borrow
-(define_insn "add<mode>3_carry"
+(define_insn "@add<mode>3_carry"
 [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(plus:SWI
 	  (plus:SWI
 	    (match_operator:SWI 4 "ix86_carry_flag_operator"
 	     [(match_operand 3 "flags_reg_operand") (const_int 0)])
 	    (match_operand:SWI 1 "nonimmediate_operand" "%0,0"))
-	  (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
+	  (match_operand:SWI 2 "<general_operand>" "<r><i>,m")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 "adc{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "use_carry" "1")
 	  (zero_extend:<DWI>
 	    (plus:SWI48
 	      (plus:SWI48
 		(match_operator:SWI48 5 "ix86_carry_flag_operator"
 		  [(match_operand 3 "flags_reg_operand") (const_int 0)])
-		(match_operand:SWI48 1 "nonimmediate_operand" "%0"))
+		(match_operand:SWI48 1 "nonimmediate_operand" "%0,0"))
-	      (match_operand:SWI48 2 "nonimmediate_operand" "rm")))
+	      (match_operand:SWI48 2 "nonimmediate_operand" "r,rm")))
 	  (plus:<DWI>
 	    (zero_extend:<DWI> (match_dup 2))
 	    (match_operator:<DWI> 4 "ix86_carry_flag_operator"
 	      [(match_dup 3) (const_int 0)]))))
-(set (match_operand:SWI48 0 "register_operand" "=r")
+(set (match_operand:SWI48 0 "nonimmediate_operand" "=rm,r")
 	(plus:SWI48 (plus:SWI48 (match_op_dup 5
 				 [(match_dup 3) (const_int 0)])
 				(match_dup 1))
 		    (match_dup 2)))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 	   (compare:CCC
 	     (plus:SWI48
 	       (match_operand:SWI48 1 "nonimmediate_operand")
 	       (match_operand:SWI48 2 "x86_64_general_operand"))
 	     (match_dup 1)))
-(set (match_operand:SWI48 0 "register_operand")
+(set (match_operand:SWI48 0 "nonimmediate_operand")
 	   (plus:SWI48 (match_dup 1) (match_dup 2)))])]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)")
-(define_insn "sub<mode>3_carry"
+(define_insn "*addcarry<mode>_1"
+[(set (reg:CCC FLAGS_REG)
+	(compare:CCC
+	  (zero_extend:<DWI>
+	    (plus:SWI48
+	      (plus:SWI48
+		(match_operator:SWI48 5 "ix86_carry_flag_operator"
+		  [(match_operand 3 "flags_reg_operand") (const_int 0)])
+		(match_operand:SWI48 1 "nonimmediate_operand" "%0"))
+	      (match_operand:SWI48 2 "x86_64_immediate_operand" "e")))
+	  (plus:<DWI>
+	    (match_operand:<DWI> 6 "const_scalar_int_operand" "")
+	    (match_operator:<DWI> 4 "ix86_carry_flag_operator"
+	      [(match_dup 3) (const_int 0)]))))
+(set (match_operand:SWI48 0 "nonimmediate_operand" "=rm")
+	(plus:SWI48 (plus:SWI48 (match_op_dup 5
+				 [(match_dup 3) (const_int 0)])
+				(match_dup 1))
+		    (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
+&& CONST_INT_P (operands[2])
+/* Check that operands[6] is operands[2] zero extended from
+<MODE>mode to <DWI>mode.  */
+&& ((<MODE>mode == SImode || INTVAL (operands[2]) >= 0)
+? (CONST_INT_P (operands[6])
+	  && UINTVAL (operands[6]) == (UINTVAL (operands[2])
+				       & GET_MODE_MASK (<MODE>mode)))
+: (CONST_WIDE_INT_P (operands[6])
+	  && CONST_WIDE_INT_NUNITS (operands[6]) == 2
+	  && ((unsigned HOST_WIDE_INT) CONST_WIDE_INT_ELT (operands[6], 0)
+	      == UINTVAL (operands[2]))
+	  && CONST_WIDE_INT_ELT (operands[6], 1) == 0))"
+"adc{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "use_carry" "1")
+(set_attr "pent_pair" "pu")
+(set_attr "mode" "<MODE>")
+(set (attr "length_immediate")
+(if_then_else (match_test "IN_RANGE (INTVAL (operands[2]), -128, 127)")
+(const_string "1")
+(const_string "4")))])
+(define_insn "@sub<mode>3_carry"
 [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(minus:SWI
 	  (minus:SWI
 	    (match_operand:SWI 1 "nonimmediate_operand" "0,0")
 	    (match_operator:SWI 4 "ix86_carry_flag_operator"
 	     [(match_operand 3 "flags_reg_operand") (const_int 0)]))
-	  (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
+	  (match_operand:SWI 2 "<general_operand>" "<r><i>,m")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
 "sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "use_carry" "1")
 [(set_attr "type" "alu")
 (set_attr "use_carry" "1")
 (set_attr "pent_pair" "pu")
 (set_attr "mode" "SI")])
-(define_insn "sub<mode>3_carry_ccc"
+(define_insn "@sub<mode>3_carry_ccc"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	  (zero_extend:<DWI> (match_operand:DWIH 1 "register_operand" "0"))
 	  (plus:<DWI>
 	    (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0))
 ;; The sign flag is set from the
 ;; (compare (match_dup 1) (plus:DWIH (ltu:DWIH ...) (match_dup 2)))
 ;; result, the overflow flag likewise, but the overflow flag is also
 ;; set if the (plus:DWIH (ltu:DWIH ...) (match_dup 2)) overflows.
-(define_insn "sub<mode>3_carry_ccgz"
+(define_insn "@sub<mode>3_carry_ccgz"
 [(set (reg:CCGZ FLAGS_REG)
 	(unspec:CCGZ [(match_operand:DWIH 1 "register_operand" "0")
 		      (match_operand:DWIH 2 "x86_64_general_operand" "rme")
 		      (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))]
 		     UNSPEC_SBB))
 (define_insn "*add<mode>3_cc_overflow_1"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	    (plus:SWI
 		(match_operand:SWI 1 "nonimmediate_operand" "%0,0")
-		(match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))
+		(match_operand:SWI 2 "<general_operand>" "<r><i>,m"))
 	    (match_dup 1)))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(plus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 "add{<imodesuffix>}\t{%2, %0|%0, %2}"
 (define_insn "*add<mode>3_cc_overflow_2"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	    (plus:SWI
 		(match_operand:SWI 1 "nonimmediate_operand" "%0,0")
-		(match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))
+		(match_operand:SWI 2 "<general_operand>" "<r><i>,m"))
 	    (match_dup 2)))
 (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(plus:SWI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
 "add{<imodesuffix>}\t{%2, %0|%0, %2}"
 	(zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
 "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
 "add{l}\t{%2, %k0|%k0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "SI")])
+(define_insn_and_split "*add<dwi>3_doubleword_cc_overflow_1"
+[(set (reg:CCC FLAGS_REG)
+	(compare:CCC
+	  (plus:<DWI>
+	    (match_operand:<DWI> 1 "nonimmediate_operand" "%0,0")
+	    (match_operand:<DWI> 2 "x86_64_hilo_general_operand" "r<di>,o"))
+	  (match_dup 1)))
+(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro,r")
+	(plus:<DWI> (match_dup 1) (match_dup 2)))]
+"ix86_binary_operator_ok (PLUS, <DWI>mode, operands)"
+"#"
+"reload_completed"
+[(parallel [(set (reg:CCC FLAGS_REG)
+		   (compare:CCC
+		     (plus:DWIH (match_dup 1) (match_dup 2))
+		     (match_dup 1)))
+	      (set (match_dup 0)
+		   (plus:DWIH (match_dup 1) (match_dup 2)))])
+(parallel [(set (reg:CCC FLAGS_REG)
+		   (compare:CCC
+		     (zero_extend:<DWI>
+		       (plus:DWIH
+			 (plus:DWIH
+			   (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+			   (match_dup 4))
+			 (match_dup 5)))
+		     (plus:<DWI>
+		       (match_dup 6)
+		       (ltu:<DWI> (reg:CC FLAGS_REG) (const_int 0)))))
+	      (set (match_dup 3)
+		   (plus:DWIH
+		     (plus:DWIH (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+				(match_dup 4))
+		     (match_dup 5)))])]
+{
+split_double_mode (<DWI>mode, &operands[0], 3, &operands[0], &operands[3]);
+if (operands[2] == const0_rtx)
+{
+emit_insn (gen_addcarry<mode>_0 (operands[3], operands[4], operands[5]));
+DONE;
+}
+if (CONST_INT_P (operands[5]))
+operands[6] = simplify_unary_operation (ZERO_EXTEND, <DWI>mode,
+					    operands[5], <MODE>mode);
+else
+operands[6] = gen_rtx_ZERO_EXTEND (<DWI>mode, operands[5]);
+})
+;; x == 0 with zero flag test can be done also as x < 1U with carry flag
+;; test, where the latter is preferrable if we have some carry consuming
+;; instruction.
+;; For x != 0, we need to use x < 1U with negation of carry, i.e.
+;; + (1 - CF).
+(define_insn_and_split "*add<mode>3_eq"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (plus:SWI
+	    (eq:SWI (match_operand 3 "int_nonimmediate_operand") (const_int 0))
+	    (match_operand:SWI 1 "nonimmediate_operand"))
+	  (match_operand:SWI 2 "<general_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 3) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (plus:SWI
+		     (plus:SWI (ltu:SWI (reg:CC FLAGS_REG) (const_int 0))
+			       (match_dup 1))
+		     (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])])
+(define_insn_and_split "*add<mode>3_ne"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (plus:SWI
+	    (ne:SWI (match_operand 3 "int_nonimmediate_operand") (const_int 0))
+	    (match_operand:SWI 1 "nonimmediate_operand"))
+	  (match_operand:SWI 2 "<immediate_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"CONST_INT_P (operands[2])
+&& (<MODE>mode != DImode
+|| INTVAL (operands[2]) != HOST_WIDE_INT_C (-0x80000000))
+&& ix86_binary_operator_ok (PLUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 3) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (minus:SWI
+		     (minus:SWI (match_dup 1)
+				(ltu:SWI (reg:CC FLAGS_REG) (const_int 0)))
+		     (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[2] = gen_int_mode (~INTVAL (operands[2]),
+			      <MODE>mode == DImode ? SImode : <MODE>mode);
+})
+(define_insn_and_split "*add<mode>3_eq_0"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (eq:SWI (match_operand 2 "int_nonimmediate_operand") (const_int 0))
+	  (match_operand:SWI 1 "<general_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_unary_operator_ok (PLUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 2) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (plus:SWI (ltu:SWI (reg:CC FLAGS_REG) (const_int 0))
+			     (match_dup 1)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+if (!nonimmediate_operand (operands[1], <MODE>mode))
+operands[1] = force_reg (<MODE>mode, operands[1]);
+})
+(define_insn_and_split "*add<mode>3_ne_0"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (ne:SWI (match_operand 2 "int_nonimmediate_operand") (const_int 0))
+	  (match_operand:SWI 1 "<general_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_unary_operator_ok (PLUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 2) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (minus:SWI (minus:SWI
+				(match_dup 1)
+				(ltu:SWI (reg:CC FLAGS_REG) (const_int 0)))
+			      (const_int -1)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+if (!nonimmediate_operand (operands[1], <MODE>mode))
+operands[1] = force_reg (<MODE>mode, operands[1]);
+})
+(define_insn_and_split "*sub<mode>3_eq"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(minus:SWI
+	  (minus:SWI
+	    (match_operand:SWI 1 "nonimmediate_operand")
+	    (eq:SWI (match_operand 3 "int_nonimmediate_operand")
+		    (const_int 0)))
+	  (match_operand:SWI 2 "<general_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 3) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (minus:SWI
+		     (minus:SWI (match_dup 1)
+				(ltu:SWI (reg:CC FLAGS_REG) (const_int 0)))
+		     (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])])
+(define_insn_and_split "*sub<mode>3_ne"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (minus:SWI
+	    (match_operand:SWI 1 "nonimmediate_operand")
+	    (ne:SWI (match_operand 3 "int_nonimmediate_operand")
+		    (const_int 0)))
+	  (match_operand:SWI 2 "<immediate_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"CONST_INT_P (operands[2])
+&& (<MODE>mode != DImode
+|| INTVAL (operands[2]) != HOST_WIDE_INT_C (-0x80000000))
+&& ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 3) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (plus:SWI
+		     (plus:SWI (ltu:SWI (reg:CC FLAGS_REG) (const_int 0))
+			       (match_dup 1))
+		     (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[2] = gen_int_mode (INTVAL (operands[2]) - 1,
+			      <MODE>mode == DImode ? SImode : <MODE>mode);
+})
+(define_insn_and_split "*sub<mode>3_eq_1"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(plus:SWI
+	  (minus:SWI
+	    (match_operand:SWI 1 "nonimmediate_operand")
+	    (eq:SWI (match_operand 3 "int_nonimmediate_operand")
+		    (const_int 0)))
+	  (match_operand:SWI 2 "<immediate_operand>")))
+(clobber (reg:CC FLAGS_REG))]
+"CONST_INT_P (operands[2])
+&& (<MODE>mode != DImode
+|| INTVAL (operands[2]) != HOST_WIDE_INT_C (-0x80000000))
+&& ix86_binary_operator_ok (MINUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 3) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (minus:SWI
+		     (minus:SWI (match_dup 1)
+				(ltu:SWI (reg:CC FLAGS_REG) (const_int 0)))
+		     (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[2] = gen_int_mode (-INTVAL (operands[2]),
+			      <MODE>mode == DImode ? SImode : <MODE>mode);
+})
+(define_insn_and_split "*sub<mode>3_eq_0"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(minus:SWI
+	  (match_operand:SWI 1 "<general_operand>")
+	  (eq:SWI (match_operand 2 "int_nonimmediate_operand") (const_int 0))))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_unary_operator_ok (MINUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 2) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (minus:SWI (match_dup 1)
+			      (ltu:SWI (reg:CC FLAGS_REG) (const_int 0))))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+if (!nonimmediate_operand (operands[1], <MODE>mode))
+operands[1] = force_reg (<MODE>mode, operands[1]);
+})
+(define_insn_and_split "*sub<mode>3_ne_0"
+[(set (match_operand:SWI 0 "nonimmediate_operand")
+	(minus:SWI
+	  (match_operand:SWI 1 "<general_operand>")
+	  (ne:SWI (match_operand 2 "int_nonimmediate_operand") (const_int 0))))
+(clobber (reg:CC FLAGS_REG))]
+"ix86_unary_operator_ok (MINUS, <MODE>mode, operands)
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (reg:CC FLAGS_REG)
+	(compare:CC (match_dup 2) (const_int 1)))
+(parallel [(set (match_dup 0)
+		   (plus:SWI (plus:SWI
+			       (ltu:SWI (reg:CC FLAGS_REG) (const_int 0))
+			       (match_dup 1))
+			     (const_int -1)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+if (!nonimmediate_operand (operands[1], <MODE>mode))
+operands[1] = force_reg (<MODE>mode, operands[1]);
+})
 ;; The patterns that match these are at the end of this file.
 (define_expand "<plusminus_insn>xf3"
 [(set (match_operand:XF 0 "register_operand")
 }
 })
 ;; Divmod instructions.
-(define_expand "divmod<mode>4"
+(define_code_iterator any_div [div udiv])
+(define_code_attr paired_mod [(div "mod") (udiv "umod")])
+(define_expand "<u>divmod<mode>4"
 [(parallel [(set (match_operand:SWIM248 0 "register_operand")
-		   (div:SWIM248
+		   (any_div:SWIM248
 		     (match_operand:SWIM248 1 "register_operand")
 		     (match_operand:SWIM248 2 "nonimmediate_operand")))
 	      (set (match_operand:SWIM248 3 "register_operand")
-		   (mod:SWIM248 (match_dup 1) (match_dup 2)))
+		   (<paired_mod>:SWIM248 (match_dup 1) (match_dup 2)))
 	      (clobber (reg:CC FLAGS_REG))])])
 ;; Split with 8bit unsigned divide:
 ;; 	if (dividend an divisor are in [0-255])
 ;;	   use 8bit unsigned integer divide
 ;;	 else
 ;;	   use original integer divide
 (define_split
 [(set (match_operand:SWI48 0 "register_operand")
-	(div:SWI48 (match_operand:SWI48 2 "register_operand")
+	(any_div:SWI48 (match_operand:SWI48 2 "register_operand")
-		    (match_operand:SWI48 3 "nonimmediate_operand")))
+		       (match_operand:SWI48 3 "nonimmediate_operand")))
 (set (match_operand:SWI48 1 "register_operand")
-	(mod:SWI48 (match_dup 2) (match_dup 3)))
+	(<paired_mod>:SWI48 (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_8BIT_IDIV
 && TARGET_QIMODE_MATH
 && can_create_pseudo_p ()
 && !optimize_insn_for_size_p ()"
 [(const_int 0)]
-"ix86_split_idivmod (<MODE>mode, operands, true); DONE;")
+"ix86_split_idivmod (<MODE>mode, operands, <u_bool>); DONE;")
 (define_split
 [(set (match_operand:DI 0 "register_operand")
 	(zero_extend:DI
-	  (div:SI (match_operand:SI 2 "register_operand")
+	  (any_div:SI (match_operand:SI 2 "register_operand")
-		  (match_operand:SI 3 "nonimmediate_operand"))))
+		      (match_operand:SI 3 "nonimmediate_operand"))))
 (set (match_operand:SI 1 "register_operand")
-	(mod:SI (match_dup 2) (match_dup 3)))
+	(<paired_mod>:SI (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
-"TARGET_USE_8BIT_IDIV
+"TARGET_64BIT
+&& TARGET_USE_8BIT_IDIV
 && TARGET_QIMODE_MATH
 && can_create_pseudo_p ()
 && !optimize_insn_for_size_p ()"
 [(const_int 0)]
-"ix86_split_idivmod (SImode, operands, true); DONE;")
+"ix86_split_idivmod (SImode, operands, <u_bool>); DONE;")
 (define_split
 [(set (match_operand:DI 1 "register_operand")
 	(zero_extend:DI
-	  (mod:SI (match_operand:SI 2 "register_operand")
+	  (<paired_mod>:SI (match_operand:SI 2 "register_operand")
-		  (match_operand:SI 3 "nonimmediate_operand"))))
+			   (match_operand:SI 3 "nonimmediate_operand"))))
 (set (match_operand:SI 0 "register_operand")
-	(div:SI  (match_dup 2) (match_dup 3)))
+	(any_div:SI  (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
-"TARGET_USE_8BIT_IDIV
+"TARGET_64BIT
+&& TARGET_USE_8BIT_IDIV
 && TARGET_QIMODE_MATH
 && can_create_pseudo_p ()
 && !optimize_insn_for_size_p ()"
 [(const_int 0)]
-"ix86_split_idivmod (SImode, operands, true); DONE;")
+"ix86_split_idivmod (SImode, operands, <u_bool>); DONE;")
 (define_insn_and_split "divmod<mode>4_1"
 [(set (match_operand:SWI48 0 "register_operand" "=a")
 	(div:SWI48 (match_operand:SWI48 2 "register_operand" "0")
 		   (match_operand:SWI48 3 "nonimmediate_operand" "rm")))
 operands[4] = operands[1];
 }
 }
 [(set_attr "type" "multi")
 (set_attr "mode" "<MODE>")])
-(define_insn_and_split "divmodsi4_zext_1"
-[(set (match_operand:DI 0 "register_operand" "=a")
-	(zero_extend:DI
-	  (div:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 1 "register_operand" "=&d")
-	(mod:SI (match_dup 2) (match_dup 3)))
-(unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"#"
-"reload_completed"
-[(parallel [(set (match_dup 1)
-		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 0)
-		   (zero_extend:DI (div:SI (match_dup 2) (match_dup 3))))
-	      (set (match_dup 1)
-		   (mod:SI (match_dup 2) (match_dup 3)))
-	      (use (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
-if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
-operands[4] = operands[2];
-else
-{
-/* Avoid use of cltd in favor of a mov+shift.  */
-emit_move_insn (operands[1], operands[2]);
-operands[4] = operands[1];
-}
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "SI")])
-(define_insn_and_split "divmodsi4_zext_2"
-[(set (match_operand:DI 1 "register_operand" "=&d")
-	(zero_extend:DI
-	  (mod:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 0 "register_operand" "=a")
-	(div:SI (match_dup 2) (match_dup 3)))
-(unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"#"
-"reload_completed"
-[(parallel [(set (match_dup 6)
-		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 1)
-		   (zero_extend:DI (mod:SI (match_dup 2) (match_dup 3))))
-	      (set (match_dup 0)
-		   (div:SI (match_dup 2) (match_dup 3)))
-	      (use (match_dup 6))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
-operands[6] = gen_lowpart (SImode, operands[1]);
-if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
-operands[4] = operands[2];
-else
-{
-/* Avoid use of cltd in favor of a mov+shift.  */
-emit_move_insn (operands[6], operands[2]);
-operands[4] = operands[6];
-}
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "SI")])
-(define_insn_and_split "*divmod<mode>4"
-[(set (match_operand:SWIM248 0 "register_operand" "=a")
-	(div:SWIM248 (match_operand:SWIM248 2 "register_operand" "0")
-		    (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
-(set (match_operand:SWIM248 1 "register_operand" "=&d")
-	(mod:SWIM248 (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-""
-"#"
-"reload_completed"
-[(parallel [(set (match_dup 1)
-		   (ashiftrt:SWIM248 (match_dup 4) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 0)
-	           (div:SWIM248 (match_dup 2) (match_dup 3)))
-	      (set (match_dup 1)
-		   (mod:SWIM248 (match_dup 2) (match_dup 3)))
-	      (use (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode)-1);
-if (<MODE>mode != HImode
-&& (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD))
-operands[4] = operands[2];
-else
-{
-/* Avoid use of cltd in favor of a mov+shift.  */
-emit_move_insn (operands[1], operands[2]);
-operands[4] = operands[1];
-}
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "<MODE>")])
-(define_insn_and_split "*divmodsi4_zext_1"
-[(set (match_operand:DI 0 "register_operand" "=a")
-	(zero_extend:DI
-	  (div:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 1 "register_operand" "=&d")
-	(mod:SI (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"#"
-"reload_completed"
-[(parallel [(set (match_dup 1)
-		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 0)
-		   (zero_extend:DI (div:SI (match_dup 2) (match_dup 3))))
-	      (set (match_dup 1)
-		   (mod:SI (match_dup 2) (match_dup 3)))
-	      (use (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
-if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
-operands[4] = operands[2];
-else
-{
-/* Avoid use of cltd in favor of a mov+shift.  */
-emit_move_insn (operands[1], operands[2]);
-operands[4] = operands[1];
-}
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "SI")])
-(define_insn_and_split "*divmodsi4_zext_2"
-[(set (match_operand:DI 1 "register_operand" "=&d")
-	(zero_extend:DI
-	  (mod:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 0 "register_operand" "=a")
-	(div:SI (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"#"
-"reload_completed"
-[(parallel [(set (match_dup 6)
-		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 1)
-		   (zero_extend:DI (mod:SI (match_dup 2) (match_dup 3))))
-	      (set (match_dup 0)
-		   (div:SI (match_dup 2) (match_dup 3)))
-	      (use (match_dup 6))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
-operands[6] = gen_lowpart (SImode, operands[1]);
-if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
-operands[4] = operands[2];
-else
-{
-/* Avoid use of cltd in favor of a mov+shift.  */
-emit_move_insn (operands[6], operands[2]);
-operands[4] = operands[6];
-}
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "SI")])
-(define_insn "*divmod<mode>4_noext"
-[(set (match_operand:SWIM248 0 "register_operand" "=a")
-	(div:SWIM248 (match_operand:SWIM248 2 "register_operand" "0")
-		    (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
-(set (match_operand:SWIM248 1 "register_operand" "=d")
-	(mod:SWIM248 (match_dup 2) (match_dup 3)))
-(use (match_operand:SWIM248 4 "register_operand" "1"))
-(clobber (reg:CC FLAGS_REG))]
-""
-"idiv{<imodesuffix>}\t%3"
-[(set_attr "type" "idiv")
-(set_attr "mode" "<MODE>")])
-(define_insn "*divmodsi4_noext_zext_1"
-[(set (match_operand:DI 0 "register_operand" "=a")
-	(zero_extend:DI
-	  (div:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 1 "register_operand" "=d")
-	(mod:SI (match_dup 2) (match_dup 3)))
-(use (match_operand:SI 4 "register_operand" "1"))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"idiv{l}\t%3"
-[(set_attr "type" "idiv")
-(set_attr "mode" "SI")])
-(define_insn "*divmodsi4_noext_zext_2"
-[(set (match_operand:DI 1 "register_operand" "=d")
-	(zero_extend:DI
-	  (mod:SI (match_operand:SI 2 "register_operand" "0")
-		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
-(set (match_operand:SI 0 "register_operand" "=a")
-	(div:SI (match_dup 2) (match_dup 3)))
-(use (match_operand:SI 4 "register_operand" "1"))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT"
-"idiv{l}\t%3"
-[(set_attr "type" "idiv")
-(set_attr "mode" "SI")])
-(define_expand "divmodqi4"
-[(parallel [(set (match_operand:QI 0 "register_operand")
-		   (div:QI
-		     (match_operand:QI 1 "register_operand")
-		     (match_operand:QI 2 "nonimmediate_operand")))
-	      (set (match_operand:QI 3 "register_operand")
-		   (mod:QI (match_dup 1) (match_dup 2)))
-	      (clobber (reg:CC FLAGS_REG))])]
-"TARGET_QIMODE_MATH"
-{
-rtx div, mod;
-rtx tmp0, tmp1;
-tmp0 = gen_reg_rtx (HImode);
-tmp1 = gen_reg_rtx (HImode);
-/* Extend operands[1] to HImode.  Generate 8bit divide.  Result is in AX.  */
-emit_insn (gen_extendqihi2 (tmp1, operands[1]));
-emit_insn (gen_divmodhiqi3 (tmp0, tmp1, operands[2]));
-/* Extract remainder from AH.  */
-tmp1 = gen_rtx_ZERO_EXTRACT (SImode, tmp0, GEN_INT (8), GEN_INT (8));
-tmp1 = lowpart_subreg (QImode, tmp1, SImode);
-rtx_insn *insn = emit_move_insn (operands[3], tmp1);
-mod = gen_rtx_MOD (QImode, operands[1], operands[2]);
-set_unique_reg_note (insn, REG_EQUAL, mod);
-/* Extract quotient from AL.  */
-insn = emit_move_insn (operands[0], gen_lowpart (QImode, tmp0));
-div = gen_rtx_DIV (QImode, operands[1], operands[2]);
-set_unique_reg_note (insn, REG_EQUAL, div);
-DONE;
-})
-;; Divide AX by r/m8, with result stored in
-;; AL <- Quotient
-;; AH <- Remainder
-;; Change div/mod to HImode and extend the second argument to HImode
-;; so that mode of div/mod matches with mode of arguments.  Otherwise
-;; combine may fail.
-(define_insn "divmodhiqi3"
-[(set (match_operand:HI 0 "register_operand" "=a")
-	(ior:HI
-	  (ashift:HI
-	    (zero_extend:HI
-	      (truncate:QI
-		(mod:HI (match_operand:HI 1 "register_operand" "0")
-			(sign_extend:HI
-			  (match_operand:QI 2 "nonimmediate_operand" "qm")))))
-	    (const_int 8))
-	  (zero_extend:HI
-	    (truncate:QI
-	      (div:HI (match_dup 1) (sign_extend:HI (match_dup 2)))))))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_QIMODE_MATH"
-"idiv{b}\t%2"
-[(set_attr "type" "idiv")
-(set_attr "mode" "QI")])
-(define_expand "udivmod<mode>4"
-[(parallel [(set (match_operand:SWIM248 0 "register_operand")
-		   (udiv:SWIM248
-		     (match_operand:SWIM248 1 "register_operand")
-		     (match_operand:SWIM248 2 "nonimmediate_operand")))
-	      (set (match_operand:SWIM248 3 "register_operand")
-		   (umod:SWIM248 (match_dup 1) (match_dup 2)))
-	      (clobber (reg:CC FLAGS_REG))])])
-;; Split with 8bit unsigned divide:
-;; 	if (dividend an divisor are in [0-255])
-;;	   use 8bit unsigned integer divide
-;;	 else
-;;	   use original integer divide
-(define_split
-[(set (match_operand:SWI48 0 "register_operand")
-	(udiv:SWI48 (match_operand:SWI48 2 "register_operand")
-		    (match_operand:SWI48 3 "nonimmediate_operand")))
-(set (match_operand:SWI48 1 "register_operand")
-	(umod:SWI48 (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_USE_8BIT_IDIV
-&& TARGET_QIMODE_MATH
-&& can_create_pseudo_p ()
-&& !optimize_insn_for_size_p ()"
-[(const_int 0)]
-"ix86_split_idivmod (<MODE>mode, operands, false); DONE;")
-(define_split
-[(set (match_operand:DI 0 "register_operand")
-	(zero_extend:DI
-	  (udiv:SI (match_operand:SI 2 "register_operand")
-		   (match_operand:SI 3 "nonimmediate_operand"))))
-(set (match_operand:SI 1 "register_operand")
-	(umod:SI (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT
-&& TARGET_USE_8BIT_IDIV
-&& TARGET_QIMODE_MATH
-&& can_create_pseudo_p ()
-&& !optimize_insn_for_size_p ()"
-[(const_int 0)]
-"ix86_split_idivmod (SImode, operands, false); DONE;")
-(define_split
-[(set (match_operand:DI 1 "register_operand")
-	(zero_extend:DI
-	  (umod:SI (match_operand:SI 2 "register_operand")
-		   (match_operand:SI 3 "nonimmediate_operand"))))
-(set (match_operand:SI 0 "register_operand")
-	(udiv:SI (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT
-&& TARGET_USE_8BIT_IDIV
-&& TARGET_QIMODE_MATH
-&& can_create_pseudo_p ()
-&& !optimize_insn_for_size_p ()"
-[(const_int 0)]
-"ix86_split_idivmod (SImode, operands, false); DONE;")
 (define_insn_and_split "udivmod<mode>4_1"
 [(set (match_operand:SWI48 0 "register_operand" "=a")
 	(udiv:SWI48 (match_operand:SWI48 2 "register_operand" "0")
 		    (match_operand:SWI48 3 "nonimmediate_operand" "rm")))
 	      (clobber (reg:CC FLAGS_REG))])]
 ""
 [(set_attr "type" "multi")
 (set_attr "mode" "<MODE>")])
+(define_insn_and_split "divmodsi4_zext_1"
+[(set (match_operand:DI 0 "register_operand" "=a")
+	(zero_extend:DI
+	  (div:SI (match_operand:SI 2 "register_operand" "0")
+		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+(set (match_operand:SI 1 "register_operand" "=&d")
+	(mod:SI (match_dup 2) (match_dup 3)))
+(unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT"
+"#"
+"&& reload_completed"
+[(parallel [(set (match_dup 1)
+		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 0)
+		   (zero_extend:DI (div:SI (match_dup 2) (match_dup 3))))
+	      (set (match_dup 1)
+		   (mod:SI (match_dup 2) (match_dup 3)))
+	      (use (match_dup 1))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
+if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
+operands[4] = operands[2];
+else
+{
+/* Avoid use of cltd in favor of a mov+shift.  */
+emit_move_insn (operands[1], operands[2]);
+operands[4] = operands[1];
+}
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "SI")])
 (define_insn_and_split "udivmodsi4_zext_1"
 [(set (match_operand:DI 0 "register_operand" "=a")
 	(zero_extend:DI
 	  (udiv:SI (match_operand:SI 2 "register_operand" "0")
 		   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 	(umod:SI (match_dup 2) (match_dup 3)))
 (unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
 "#"
-"reload_completed"
+"&& reload_completed"
 [(set (match_dup 1) (const_int 0))
 (parallel [(set (match_dup 0)
 		   (zero_extend:DI (udiv:SI (match_dup 2) (match_dup 3))))
 	      (set (match_dup 1)
 		   (umod:SI (match_dup 2) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
 ""
 [(set_attr "type" "multi")
 (set_attr "mode" "SI")])
+(define_insn_and_split "divmodsi4_zext_2"
+[(set (match_operand:DI 1 "register_operand" "=&d")
+	(zero_extend:DI
+	  (mod:SI (match_operand:SI 2 "register_operand" "0")
+		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+(set (match_operand:SI 0 "register_operand" "=a")
+	(div:SI (match_dup 2) (match_dup 3)))
+(unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT"
+"#"
+"&& reload_completed"
+[(parallel [(set (match_dup 6)
+		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 1)
+		   (zero_extend:DI (mod:SI (match_dup 2) (match_dup 3))))
+	      (set (match_dup 0)
+		   (div:SI (match_dup 2) (match_dup 3)))
+	      (use (match_dup 6))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
+operands[6] = gen_lowpart (SImode, operands[1]);
+if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
+operands[4] = operands[2];
+else
+{
+/* Avoid use of cltd in favor of a mov+shift.  */
+emit_move_insn (operands[6], operands[2]);
+operands[4] = operands[6];
+}
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "SI")])
 (define_insn_and_split "udivmodsi4_zext_2"
 [(set (match_operand:DI 1 "register_operand" "=&d")
 	(zero_extend:DI
 	  (umod:SI (match_operand:SI 2 "register_operand" "0")
 		 (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 	(udiv:SI (match_dup 2) (match_dup 3)))
 (unspec [(const_int 0)] UNSPEC_DIV_ALREADY_SPLIT)
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
 "#"
-"reload_completed"
+"&& reload_completed"
 [(set (match_dup 4) (const_int 0))
 (parallel [(set (match_dup 1)
 		   (zero_extend:DI (umod:SI (match_dup 2) (match_dup 3))))
 	      (set (match_dup 0)
 		   (udiv:SI (match_dup 2) (match_dup 3)))
 	      (use (match_dup 4))
 	      (clobber (reg:CC FLAGS_REG))])]
 "operands[4] = gen_lowpart (SImode, operands[1]);"
 [(set_attr "type" "multi")
 (set_attr "mode" "SI")])
+(define_insn_and_split "*divmod<mode>4"
+[(set (match_operand:SWIM248 0 "register_operand" "=a")
+	(div:SWIM248 (match_operand:SWIM248 2 "register_operand" "0")
+		    (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
+(set (match_operand:SWIM248 1 "register_operand" "=&d")
+	(mod:SWIM248 (match_dup 2) (match_dup 3)))
+(clobber (reg:CC FLAGS_REG))]
+""
+"#"
+"reload_completed"
+[(parallel [(set (match_dup 1)
+		   (ashiftrt:SWIM248 (match_dup 4) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 0)
+	           (div:SWIM248 (match_dup 2) (match_dup 3)))
+	      (set (match_dup 1)
+		   (mod:SWIM248 (match_dup 2) (match_dup 3)))
+	      (use (match_dup 1))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode)-1);
+if (<MODE>mode != HImode
+&& (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD))
+operands[4] = operands[2];
+else
+{
+/* Avoid use of cltd in favor of a mov+shift.  */
+emit_move_insn (operands[1], operands[2]);
+operands[4] = operands[1];
+}
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "<MODE>")])
 (define_insn_and_split "*udivmod<mode>4"
 [(set (match_operand:SWIM248 0 "register_operand" "=a")
 	(udiv:SWIM248 (match_operand:SWIM248 2 "register_operand" "0")
 		      (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
 	      (clobber (reg:CC FLAGS_REG))])]
 ""
 [(set_attr "type" "multi")
 (set_attr "mode" "<MODE>")])
+;; Optimize division or modulo by constant power of 2, if the constant
+;; materializes only after expansion.
+(define_insn_and_split "*udivmod<mode>4_pow2"
+[(set (match_operand:SWI48 0 "register_operand" "=r")
+	(udiv:SWI48 (match_operand:SWI48 2 "register_operand" "0")
+		    (match_operand:SWI48 3 "const_int_operand" "n")))
+(set (match_operand:SWI48 1 "register_operand" "=r")
+	(umod:SWI48 (match_dup 2) (match_dup 3)))
+(clobber (reg:CC FLAGS_REG))]
+"IN_RANGE (exact_log2 (UINTVAL (operands[3])), 1, 31)"
+"#"
+"&& reload_completed"
+[(set (match_dup 1) (match_dup 2))
+(parallel [(set (match_dup 0) (lshiftrt:<MODE> (match_dup 2) (match_dup 4)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 1) (and:<MODE> (match_dup 1) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+int v = exact_log2 (UINTVAL (operands[3]));
+operands[4] = GEN_INT (v);
+operands[5] = GEN_INT ((HOST_WIDE_INT_1U << v) - 1);
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "<MODE>")])
+(define_insn_and_split "*divmodsi4_zext_1"
+[(set (match_operand:DI 0 "register_operand" "=a")
+	(zero_extend:DI
+	  (div:SI (match_operand:SI 2 "register_operand" "0")
+		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+(set (match_operand:SI 1 "register_operand" "=&d")
+	(mod:SI (match_dup 2) (match_dup 3)))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT"
+"#"
+"&& reload_completed"
+[(parallel [(set (match_dup 1)
+		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 0)
+		   (zero_extend:DI (div:SI (match_dup 2) (match_dup 3))))
+	      (set (match_dup 1)
+		   (mod:SI (match_dup 2) (match_dup 3)))
+	      (use (match_dup 1))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
+if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
+operands[4] = operands[2];
+else
+{
+/* Avoid use of cltd in favor of a mov+shift.  */
+emit_move_insn (operands[1], operands[2]);
+operands[4] = operands[1];
+}
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "SI")])
 (define_insn_and_split "*udivmodsi4_zext_1"
 [(set (match_operand:DI 0 "register_operand" "=a")
 	(zero_extend:DI
 	  (udiv:SI (match_operand:SI 2 "register_operand" "0")
 		   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 (set (match_operand:SI 1 "register_operand" "=&d")
 	(umod:SI (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
 "#"
-"reload_completed"
+"&& reload_completed"
 [(set (match_dup 1) (const_int 0))
 (parallel [(set (match_dup 0)
 		   (zero_extend:DI (udiv:SI (match_dup 2) (match_dup 3))))
 	      (set (match_dup 1)
 		   (umod:SI (match_dup 2) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
 ""
 [(set_attr "type" "multi")
 (set_attr "mode" "SI")])
+(define_insn_and_split "*udivmodsi4_pow2_zext_1"
+[(set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (udiv:SI (match_operand:SI 2 "register_operand" "0")
+		   (match_operand:SI 3 "const_int_operand" "n"))))
+(set (match_operand:SI 1 "register_operand" "=r")
+	(umod:SI (match_dup 2) (match_dup 3)))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT
+&& exact_log2 (UINTVAL (operands[3])) > 0"
+"#"
+"&& reload_completed"
+[(set (match_dup 1) (match_dup 2))
+(parallel [(set (match_dup 0)
+		   (zero_extend:DI (lshiftrt:SI (match_dup 2) (match_dup 4))))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 1) (and:SI (match_dup 1) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+int v = exact_log2 (UINTVAL (operands[3]));
+operands[4] = GEN_INT (v);
+operands[5] = GEN_INT ((HOST_WIDE_INT_1U << v) - 1);
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "SI")])
+(define_insn_and_split "*divmodsi4_zext_2"
+[(set (match_operand:DI 1 "register_operand" "=&d")
+	(zero_extend:DI
+	  (mod:SI (match_operand:SI 2 "register_operand" "0")
+		  (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+(set (match_operand:SI 0 "register_operand" "=a")
+	(div:SI (match_dup 2) (match_dup 3)))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT"
+"#"
+"&& reload_completed"
+[(parallel [(set (match_dup 6)
+		   (ashiftrt:SI (match_dup 4) (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])
+(parallel [(set (match_dup 1)
+		   (zero_extend:DI (mod:SI (match_dup 2) (match_dup 3))))
+	      (set (match_dup 0)
+		   (div:SI (match_dup 2) (match_dup 3)))
+	      (use (match_dup 6))
+	      (clobber (reg:CC FLAGS_REG))])]
+{
+operands[5] = GEN_INT (GET_MODE_BITSIZE (SImode)-1);
+operands[6] = gen_lowpart (SImode, operands[1]);
+if (optimize_function_for_size_p (cfun) || TARGET_USE_CLTD)
+operands[4] = operands[2];
+else
+{
+/* Avoid use of cltd in favor of a mov+shift.  */
+emit_move_insn (operands[6], operands[2]);
+operands[4] = operands[6];
+}
+}
+[(set_attr "type" "multi")
+(set_attr "mode" "SI")])
 (define_insn_and_split "*udivmodsi4_zext_2"
 [(set (match_operand:DI 1 "register_operand" "=&d")
 	(zero_extend:DI
 	  (umod:SI (match_operand:SI 2 "register_operand" "0")
 		   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 (set (match_operand:SI 0 "register_operand" "=a")
 	(udiv:SI (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
 "#"
-"reload_completed"
+"&& reload_completed"
 [(set (match_dup 4) (const_int 0))
 (parallel [(set (match_dup 1)
 		   (zero_extend:DI (umod:SI (match_dup 2) (match_dup 3))))
 	      (set (match_dup 0)
 		   (udiv:SI (match_dup 2) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
 "operands[4] = gen_lowpart (SImode, operands[1]);"
 [(set_attr "type" "multi")
 (set_attr "mode" "SI")])
-;; Optimize division or modulo by constant power of 2, if the constant
-;; materializes only after expansion.
-(define_insn_and_split "*udivmod<mode>4_pow2"
-[(set (match_operand:SWI48 0 "register_operand" "=r")
-	(udiv:SWI48 (match_operand:SWI48 2 "register_operand" "0")
-		    (match_operand:SWI48 3 "const_int_operand" "n")))
-(set (match_operand:SWI48 1 "register_operand" "=r")
-	(umod:SWI48 (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"IN_RANGE (INTVAL (operands[3]), 2, HOST_WIDE_INT_UC (0x80000000))
-&& (UINTVAL (operands[3]) & (UINTVAL (operands[3]) - 1)) == 0"
-"#"
-"&& 1"
-[(set (match_dup 1) (match_dup 2))
-(parallel [(set (match_dup 0) (lshiftrt:<MODE> (match_dup 2) (match_dup 4)))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 1) (and:<MODE> (match_dup 1) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-int v = exact_log2 (UINTVAL (operands[3]));
-operands[4] = GEN_INT (v);
-operands[5] = GEN_INT ((HOST_WIDE_INT_1U << v) - 1);
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "<MODE>")])
-(define_insn_and_split "*udivmodsi4_pow2_zext_1"
-[(set (match_operand:DI 0 "register_operand" "=r")
-	(zero_extend:DI
-	  (udiv:SI (match_operand:SI 2 "register_operand" "0")
-		   (match_operand:SI 3 "const_int_operand" "n"))))
-(set (match_operand:SI 1 "register_operand" "=r")
-	(umod:SI (match_dup 2) (match_dup 3)))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_64BIT
-&& IN_RANGE (INTVAL (operands[3]), 2, HOST_WIDE_INT_UC (0x80000000))
-&& (UINTVAL (operands[3]) & (UINTVAL (operands[3]) - 1)) == 0"
-"#"
-"&& 1"
-[(set (match_dup 1) (match_dup 2))
-(parallel [(set (match_dup 0)
-		   (zero_extend:DI (lshiftrt:SI (match_dup 2) (match_dup 4))))
-	      (clobber (reg:CC FLAGS_REG))])
-(parallel [(set (match_dup 1) (and:SI (match_dup 1) (match_dup 5)))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-int v = exact_log2 (UINTVAL (operands[3]));
-operands[4] = GEN_INT (v);
-operands[5] = GEN_INT ((HOST_WIDE_INT_1U << v) - 1);
-}
-[(set_attr "type" "multi")
-(set_attr "mode" "SI")])
 (define_insn_and_split "*udivmodsi4_pow2_zext_2"
 [(set (match_operand:DI 1 "register_operand" "=r")
 	(zero_extend:DI
 	  (umod:SI (match_operand:SI 2 "register_operand" "0")
 		   (match_operand:SI 3 "const_int_operand" "n"))))
 (set (match_operand:SI 0 "register_operand" "=r")
 	(umod:SI (match_dup 2) (match_dup 3)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT
-&& IN_RANGE (INTVAL (operands[3]), 2, HOST_WIDE_INT_UC (0x80000000))
+&& exact_log2 (UINTVAL (operands[3])) > 0"
-&& (UINTVAL (operands[3]) & (UINTVAL (operands[3]) - 1)) == 0"
 "#"
-"&& 1"
+"&& reload_completed"
 [(set (match_dup 1) (match_dup 2))
 (parallel [(set (match_dup 0) (lshiftrt:SI (match_dup 2) (match_dup 4)))
 	      (clobber (reg:CC FLAGS_REG))])
 (parallel [(set (match_dup 1)
 		   (zero_extend:DI (and:SI (match_dup 1) (match_dup 5))))
 operands[5] = GEN_INT ((HOST_WIDE_INT_1U << v) - 1);
 }
 [(set_attr "type" "multi")
 (set_attr "mode" "SI")])
-(define_insn "*udivmod<mode>4_noext"
+(define_insn "*<u>divmod<mode>4_noext"
 [(set (match_operand:SWIM248 0 "register_operand" "=a")
-	(udiv:SWIM248 (match_operand:SWIM248 2 "register_operand" "0")
+	(any_div:SWIM248
-		      (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
+	  (match_operand:SWIM248 2 "register_operand" "0")
+	  (match_operand:SWIM248 3 "nonimmediate_operand" "rm")))
 (set (match_operand:SWIM248 1 "register_operand" "=d")
-	(umod:SWIM248 (match_dup 2) (match_dup 3)))
+	(<paired_mod>:SWIM248 (match_dup 2) (match_dup 3)))
 (use (match_operand:SWIM248 4 "register_operand" "1"))
 (clobber (reg:CC FLAGS_REG))]
 ""
-"div{<imodesuffix>}\t%3"
+"<sgnprefix>div{<imodesuffix>}\t%3"
 [(set_attr "type" "idiv")
 (set_attr "mode" "<MODE>")])
-(define_insn "*udivmodsi4_noext_zext_1"
+(define_insn "*<u>divmodsi4_noext_zext_1"
 [(set (match_operand:DI 0 "register_operand" "=a")
 	(zero_extend:DI
-	  (udiv:SI (match_operand:SI 2 "register_operand" "0")
+	  (any_div:SI (match_operand:SI 2 "register_operand" "0")
-		   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+		      (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 (set (match_operand:SI 1 "register_operand" "=d")
-	(umod:SI (match_dup 2) (match_dup 3)))
+	(<paired_mod>:SI (match_dup 2) (match_dup 3)))
 (use (match_operand:SI 4 "register_operand" "1"))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
-"div{l}\t%3"
+"<sgnprefix>div{l}\t%3"
 [(set_attr "type" "idiv")
 (set_attr "mode" "SI")])
-(define_insn "*udivmodsi4_noext_zext_2"
+(define_insn "*<u>divmodsi4_noext_zext_2"
 [(set (match_operand:DI 1 "register_operand" "=d")
 	(zero_extend:DI
-	  (umod:SI (match_operand:SI 2 "register_operand" "0")
+	  (<paired_mod>:SI (match_operand:SI 2 "register_operand" "0")
-		   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
+			   (match_operand:SI 3 "nonimmediate_operand" "rm"))))
 (set (match_operand:SI 0 "register_operand" "=a")
-	(udiv:SI (match_dup 2) (match_dup 3)))
+	(any_div:SI (match_dup 2) (match_dup 3)))
 (use (match_operand:SI 4 "register_operand" "1"))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
-"div{l}\t%3"
+"<sgnprefix>div{l}\t%3"
 [(set_attr "type" "idiv")
 (set_attr "mode" "SI")])
+(define_expand "divmodqi4"
+[(parallel [(set (match_operand:QI 0 "register_operand")
+		   (div:QI
+		     (match_operand:QI 1 "register_operand")
+		     (match_operand:QI 2 "nonimmediate_operand")))
+	      (set (match_operand:QI 3 "register_operand")
+		   (mod:QI (match_dup 1) (match_dup 2)))
+	      (clobber (reg:CC FLAGS_REG))])]
+"TARGET_QIMODE_MATH"
+{
+rtx div, mod;
+rtx tmp0, tmp1;
+tmp0 = gen_reg_rtx (HImode);
+tmp1 = gen_reg_rtx (HImode);
+/* Extend operands[1] to HImode.  Generate 8bit divide.  Result is in AX.  */
+emit_insn (gen_extendqihi2 (tmp1, operands[1]));
+emit_insn (gen_divmodhiqi3 (tmp0, tmp1, operands[2]));
+/* Extract remainder from AH.  */
+tmp1 = gen_rtx_ZERO_EXTRACT (SImode, tmp0, GEN_INT (8), GEN_INT (8));
+tmp1 = lowpart_subreg (QImode, tmp1, SImode);
+rtx_insn *insn = emit_move_insn (operands[3], tmp1);
+mod = gen_rtx_MOD (QImode, operands[1], operands[2]);
+set_unique_reg_note (insn, REG_EQUAL, mod);
+/* Extract quotient from AL.  */
+insn = emit_move_insn (operands[0], gen_lowpart (QImode, tmp0));
+div = gen_rtx_DIV (QImode, operands[1], operands[2]);
+set_unique_reg_note (insn, REG_EQUAL, div);
+DONE;
+})
 (define_expand "udivmodqi4"
 [(parallel [(set (match_operand:QI 0 "register_operand")
 		   (udiv:QI
 		     (match_operand:QI 1 "register_operand")
 	      (clobber (reg:CC FLAGS_REG))])]
 "TARGET_QIMODE_MATH"
 {
 rtx div, mod;
 rtx tmp0, tmp1;
 tmp0 = gen_reg_rtx (HImode);
 tmp1 = gen_reg_rtx (HImode);
 /* Extend operands[1] to HImode.  Generate 8bit divide.  Result is in AX.  */
 emit_insn (gen_zero_extendqihi2 (tmp1, operands[1]));
 set_unique_reg_note (insn, REG_EQUAL, div);
 DONE;
 })
-(define_insn "udivmodhiqi3"
+;; Divide AX by r/m8, with result stored in
+;; AL <- Quotient
+;; AH <- Remainder
+;; Change div/mod to HImode and extend the second argument to HImode
+;; so that mode of div/mod matches with mode of arguments.  Otherwise
+;; combine may fail.
+(define_insn "<u>divmodhiqi3"
 [(set (match_operand:HI 0 "register_operand" "=a")
 	(ior:HI
 	  (ashift:HI
 	    (zero_extend:HI
 	      (truncate:QI
 		(mod:HI (match_operand:HI 1 "register_operand" "0")
-			(zero_extend:HI
+			(any_extend:HI
 			  (match_operand:QI 2 "nonimmediate_operand" "qm")))))
 	    (const_int 8))
 	  (zero_extend:HI
 	    (truncate:QI
-	      (div:HI (match_dup 1) (zero_extend:HI (match_dup 2)))))))
+	      (div:HI (match_dup 1) (any_extend:HI (match_dup 2)))))))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_QIMODE_MATH"
-"div{b}\t%2"
+"<sgnprefix>div{b}\t%2"
 [(set_attr "type" "idiv")
 (set_attr "mode" "QI")])
 ;; We cannot use div/idiv for double division, because it causes
 ;; "division by zero" on the overflow and that's not what we expect
 ;;- Logical AND instructions
 ;; On Pentium, "test imm, reg" is pairable only with eax, ax, and al.
 ;; Note that this excludes ah.
-(define_expand "testsi_ccno_1"
+(define_expand "@test<mode>_ccno_1"
 [(set (reg:CCNO FLAGS_REG)
 	(compare:CCNO
-	  (and:SI (match_operand:SI 0 "nonimmediate_operand")
+	  (and:SWI48
-		  (match_operand:SI 1 "x86_64_nonmemory_operand"))
+	    (match_operand:SWI48 0 "nonimmediate_operand")
+	    (match_operand:SWI48 1 "<nonmemory_szext_operand>"))
 	  (const_int 0)))])
 (define_expand "testqi_ccz_1"
 [(set (reg:CCZ FLAGS_REG)
-(compare:CCZ (and:QI (match_operand:QI 0 "nonimmediate_operand")
+	(compare:CCZ
-			     (match_operand:QI 1 "nonmemory_operand"))
+	  (and:QI
-		 (const_int 0)))])
+	    (match_operand:QI 0 "nonimmediate_operand")
+	    (match_operand:QI 1 "nonmemory_operand"))
-(define_expand "testdi_ccno_1"
+	  (const_int 0)))])
-[(set (reg:CCNO FLAGS_REG)
-	(compare:CCNO
-	  (and:DI (match_operand:DI 0 "nonimmediate_operand")
-		  (match_operand:DI 1 "x86_64_szext_general_operand"))
-	  (const_int 0)))]
-"TARGET_64BIT && !(MEM_P (operands[0]) && MEM_P (operands[1]))")
 (define_insn "*testdi_1"
 [(set (reg FLAGS_REG)
 	(compare
-	 (and:DI
+	  (and:DI
-	  (match_operand:DI 0 "nonimmediate_operand" "%!*a,r,!*a,r,rm")
+	    (match_operand:DI 0 "nonimmediate_operand" "%r,rm")
-	  (match_operand:DI 1 "x86_64_szext_general_operand" "Z,Z,e,e,re"))
+	    (match_operand:DI 1 "x86_64_szext_nonmemory_operand" "Z,re"))
 	 (const_int 0)))]
-"TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+"TARGET_64BIT
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+&& ix86_match_ccmode
+	(insn,
+	 /* If we are going to emit testl instead of testq, and the operands[1]
+	    constant might have the SImode sign bit set, make sure the sign
+	    flag isn't tested, because the instruction will set the sign flag
+	    based on bit 31 rather than bit 63.  If it isn't CONST_INT,
+	    conservatively assume it might have bit 31 set.  */
+	 (satisfies_constraint_Z (operands[1])
+	  && (!CONST_INT_P (operands[1])
+	      || val_signbit_known_set_p (SImode, INTVAL (operands[1]))))
+	 ? CCZmode : CCNOmode)"
 "@
 test{l}\t{%k1, %k0|%k0, %k1}
-test{l}\t{%k1, %k0|%k0, %k1}
-test{q}\t{%1, %0|%0, %1}
-test{q}\t{%1, %0|%0, %1}
 test{q}\t{%1, %0|%0, %1}"
 [(set_attr "type" "test")
-(set_attr "modrm" "0,1,0,1,1")
+(set_attr "mode" "SI,DI")])
-(set_attr "mode" "SI,SI,DI,DI,DI")])
 (define_insn "*testqi_1_maybe_si"
 [(set (reg FLAGS_REG)
-(compare
+	(compare
 	  (and:QI
-	    (match_operand:QI 0 "nonimmediate_operand" "%!*a,q,qm,r")
+	    (match_operand:QI 0 "nonimmediate_operand" "%qm,*a,qm,r")
-	    (match_operand:QI 1 "general_operand" "n,n,qn,n"))
+	    (match_operand:QI 1 "nonmemory_operand" "q,n,n,n"))
 	  (const_int 0)))]
-"!(MEM_P (operands[0]) && MEM_P (operands[1]))
+"ix86_match_ccmode (insn,
-&& ix86_match_ccmode (insn,
+		      CONST_INT_P (operands[1])
-			 CONST_INT_P (operands[1])
+		      && INTVAL (operands[1]) >= 0 ? CCNOmode : CCZmode)"
-			 && INTVAL (operands[1]) >= 0 ? CCNOmode : CCZmode)"
 {
 if (which_alternative == 3)
 {
 if (CONST_INT_P (operands[1]) && INTVAL (operands[1]) < 0)
 	operands[1] = GEN_INT (INTVAL (operands[1]) & 0xff);
 return "test{l}\t{%1, %k0|%k0, %1}";
 }
 return "test{b}\t{%1, %0|%0, %1}";
 }
 [(set_attr "type" "test")
-(set_attr "modrm" "0,1,1,1")
 (set_attr "mode" "QI,QI,QI,SI")
-(set_attr "pent_pair" "uv,np,uv,np")])
+(set_attr "pent_pair" "uv,uv,np,np")])
 (define_insn "*test<mode>_1"
 [(set (reg FLAGS_REG)
 	(compare
-	 (and:SWI124
+	  (and:SWI124
-	  (match_operand:SWI124 0 "nonimmediate_operand" "%!*a,<r>,<r>m")
+	    (match_operand:SWI124 0 "nonimmediate_operand" "%<r>m,*a,<r>m")
-	  (match_operand:SWI124 1 "<general_operand>" "<i>,<i>,<r><i>"))
+	    (match_operand:SWI124 1 "<nonmemory_szext_operand>" "<r>,<i>,<i>"))
 	 (const_int 0)))]
-"ix86_match_ccmode (insn, CCNOmode)
+"ix86_match_ccmode (insn, CCNOmode)"
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
 "test{<imodesuffix>}\t{%1, %0|%0, %1}"
 [(set_attr "type" "test")
-(set_attr "modrm" "0,1,1")
 (set_attr "mode" "<MODE>")
-(set_attr "pent_pair" "uv,np,uv")])
+(set_attr "pent_pair" "uv,uv,np")])
 (define_expand "testqi_ext_1_ccno"
 [(set (reg:CCNO FLAGS_REG)
 	(compare:CCNO
 	  (and:QI
 ;; %%% This used to optimize known byte-wide and operations to memory,
 ;; and sometimes to QImode registers.  If this is considered useful,
 ;; it should be done with splitters.
 (define_expand "and<mode>3"
-[(set (match_operand:SWIM1248x 0 "nonimmediate_operand")
+[(set (match_operand:SWIM1248s 0 "nonimmediate_operand")
-	(and:SWIM1248x (match_operand:SWIM1248x 1 "nonimmediate_operand")
+	(and:SWIM1248s (match_operand:SWIM1248s 1 "nonimmediate_operand")
-		      (match_operand:SWIM1248x 2 "<general_szext_operand>")))]
+		       (match_operand:SWIM1248s 2 "<general_szext_operand>")))]
 ""
 {
 machine_mode mode = <MODE>mode;
-rtx (*insn) (rtx, rtx);
+if (<MODE>mode == DImode && !TARGET_64BIT)
-if (CONST_INT_P (operands[2]) && REG_P (operands[0]))
+;
+else if (const_int_operand (operands[2], <MODE>mode)
+	   && register_operand (operands[0], <MODE>mode)
+	   && !(TARGET_ZERO_EXTEND_WITH_AND
+		&& optimize_function_for_speed_p (cfun)))
 {
-HOST_WIDE_INT ival = INTVAL (operands[2]);
+unsigned HOST_WIDE_INT ival = UINTVAL (operands[2]);
-if (ival == (HOST_WIDE_INT) 0xffffffff)
+if (ival == GET_MODE_MASK (SImode))
 	mode = SImode;
-else if (ival == 0xffff)
+else if (ival == GET_MODE_MASK (HImode))
 	mode = HImode;
-else if (ival == 0xff)
+else if (ival == GET_MODE_MASK (QImode))
 	mode = QImode;
-}
-if (mode == <MODE>mode)
-{
-ix86_expand_binary_operator (AND, <MODE>mode, operands);
-DONE;
 }
-if (<MODE>mode == DImode)
+if (mode != <MODE>mode)
-insn = (mode == SImode)
+emit_insn (gen_extend_insn
-	   ? gen_zero_extendsidi2
+	       (operands[0], gen_lowpart (mode, operands[1]),
-	   : (mode == HImode)
+		<MODE>mode, mode, 1));
-	   ? gen_zero_extendhidi2
-	   : gen_zero_extendqidi2;
-else if (<MODE>mode == SImode)
-insn = (mode == HImode)
-	   ? gen_zero_extendhisi2
-	   : gen_zero_extendqisi2;
-else if (<MODE>mode == HImode)
-insn = gen_zero_extendqihi2;
 else
-gcc_unreachable ();
+ix86_expand_binary_operator (AND, <MODE>mode, operands);
-emit_insn (insn (operands[0], gen_lowpart (mode, operands[1])));
 DONE;
 })
 (define_insn_and_split "*anddi3_doubleword"
-[(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r")
+[(set (match_operand:DI 0 "nonimmediate_operand")
 	(and:DI
-	 (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
+	 (match_operand:DI 1 "nonimmediate_operand")
-	 (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm")))
+	 (match_operand:DI 2 "x86_64_szext_general_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-&& ix86_binary_operator_ok (AND, DImode, operands)"
+&& ix86_binary_operator_ok (AND, DImode, operands)
+&& ix86_pre_reload_split ()"
 "#"
-"&& reload_completed"
+"&& 1"
 [(const_int 0)]
 {
 split_double_mode (DImode, &operands[0], 3, &operands[0], &operands[3]);
 if (operands[2] == const0_rtx)
-{
+emit_move_insn (operands[0], const0_rtx);
-operands[1] = const0_rtx;
+else if (operands[2] == constm1_rtx)
-ix86_expand_move (SImode, &operands[0]);
+emit_move_insn (operands[0], operands[1]);
-}
+else
-else if (operands[2] != constm1_rtx)
+emit_insn (gen_andsi3 (operands[0], operands[1], operands[2]));
-ix86_expand_binary_operator (AND, SImode, &operands[0]);
+if (operands[5] == const0_rtx)
+emit_move_insn (operands[3], const0_rtx);
 else if (operands[5] == constm1_rtx)
-emit_note (NOTE_INSN_DELETED);
+emit_move_insn (operands[3], operands[4]);
-if (operands[5] == const0_rtx)
+else
-{
+emit_insn (gen_andsi3 (operands[3], operands[4], operands[5]));
-operands[4] = const0_rtx;
-ix86_expand_move (SImode, &operands[3]);
-}
-else if (operands[5] != constm1_rtx)
-ix86_expand_binary_operator (AND, SImode, &operands[3]);
 DONE;
 })
 (define_insn "*anddi_1"
 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r,r")
 	(and:DI
 	 (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,qm")
-	 (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm,L")))
+	 (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,m,L")))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT && ix86_binary_operator_ok (AND, DImode, operands)"
 "@
 and{l}\t{%k2, %k0|%k0, %k2}
 and{q}\t{%2, %0|%0, %2}
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT"
 [(parallel [(set (match_dup 0)
 		   (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))
 	      (clobber (reg:CC FLAGS_REG))])]
-"operands[2] = gen_lowpart (SImode, operands[2]);")
+{
+if (GET_CODE (operands[2]) == SYMBOL_REF
+|| GET_CODE (operands[2]) == LABEL_REF)
+{
+operands[2] = shallow_copy_rtx (operands[2]);
+PUT_MODE (operands[2], SImode);
+}
+else if (GET_CODE (operands[2]) == CONST)
+{
+/* (const:DI (plus:DI (symbol_ref:DI ("...")) (const_int N))) */
+operands[2] = copy_rtx (operands[2]);
+PUT_MODE (operands[2], SImode);
+PUT_MODE (XEXP (operands[2], 0), SImode);
+PUT_MODE (XEXP (XEXP (operands[2], 0), 0), SImode);
+}
+else
+operands[2] = gen_lowpart (SImode, operands[2]);
+})
 ;; See comment for addsi_1_zext why we do use nonimmediate_operand
 (define_insn "*andsi_1_zext"
 [(set (match_operand:DI 0 "register_operand" "=r")
 	(zero_extend:DI
 (set_attr "mode" "SI")])
 (define_insn "*and<mode>_1"
 [(set (match_operand:SWI24 0 "nonimmediate_operand" "=rm,r,Ya")
 	(and:SWI24 (match_operand:SWI24 1 "nonimmediate_operand" "%0,0,qm")
-		   (match_operand:SWI24 2 "<general_operand>" "r<i>,rm,L")))
+		   (match_operand:SWI24 2 "<general_operand>" "r<i>,m,L")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (AND, <MODE>mode, operands)"
 "@
 and{<imodesuffix>}\t{%2, %0|%0, %2}
 and{<imodesuffix>}\t{%2, %0|%0, %2}
 (set_attr "mode" "<MODE>,<MODE>,SI")])
 (define_insn "*andqi_1"
 [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,r")
 	(and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-		(match_operand:QI 2 "general_operand" "qn,qmn,rn")))
+		(match_operand:QI 2 "general_operand" "qn,m,rn")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (AND, QImode, operands)"
 "@
 and{b}\t{%2, %0|%0, %2}
 and{b}\t{%2, %0|%0, %2}
 (set (attr "preferred_for_speed")
 (cond [(eq_attr "alternative" "2")
 	      (symbol_ref "!TARGET_PARTIAL_REG_STALL")]
 	   (symbol_ref "true")))])
-(define_insn "*andqi_1_slp"
+(define_insn "*and<mode>_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(and:QI (match_dup 0)
+	(and:SWI12 (match_operand:SWI12 1 "nonimmediate_operand" "%0")
-		(match_operand:QI 1 "general_operand" "qn,qmn")))
+		   (match_operand:SWI12 2 "general_operand" "<r>mn")))
 (clobber (reg:CC FLAGS_REG))]
 "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-"and{b}\t{%1, %0|%0, %1}"
+&& (rtx_equal_p (operands[0], operands[1])
-[(set_attr "type" "alu1")
+|| rtx_equal_p (operands[0], operands[2]))"
-(set_attr "mode" "QI")])
+"and{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")])
 (define_split
 [(set (match_operand:SWI248 0 "register_operand")
 	(and:SWI248 (match_operand:SWI248 1 "nonimmediate_operand")
 		    (match_operand:SWI248 2 "const_int_operand")))
 "reload_completed
 && (!REG_P (operands[1])
 || REGNO (operands[0]) != REGNO (operands[1]))"
 [(const_int 0)]
 {
-HOST_WIDE_INT ival = INTVAL (operands[2]);
+unsigned HOST_WIDE_INT ival = UINTVAL (operands[2]);
 machine_mode mode;
-rtx (*insn) (rtx, rtx);
+if (ival == GET_MODE_MASK (SImode))
-if (ival == (HOST_WIDE_INT) 0xffffffff)
 mode = SImode;
-else if (ival == 0xffff)
+else if (ival == GET_MODE_MASK (HImode))
 mode = HImode;
+else if (ival == GET_MODE_MASK (QImode))
+mode = QImode;
 else
-{
+gcc_unreachable ();
-gcc_assert (ival == 0xff);
-mode = QImode;
+/* Zero extend to SImode to avoid partial register stalls.  */
-}
+if (<MODE_SIZE> < GET_MODE_SIZE (SImode))
+operands[0] = gen_lowpart (SImode, operands[0]);
-if (<MODE>mode == DImode)
-insn = (mode == SImode)
+emit_insn (gen_extend_insn
-	   ? gen_zero_extendsidi2
+	     (operands[0], gen_lowpart (mode, operands[1]),
-	   : (mode == HImode)
+	      GET_MODE (operands[0]), mode, 1));
-	   ? gen_zero_extendhidi2
-	   : gen_zero_extendqidi2;
-else
-{
-if (<MODE>mode != SImode)
-	/* Zero extend to SImode to avoid partial register stalls.  */
-	operands[0] = gen_lowpart (SImode, operands[0]);
-insn = (mode == HImode)
-	     ? gen_zero_extendhisi2
-	     : gen_zero_extendqisi2;
-}
-emit_insn (insn (operands[0], gen_lowpart (mode, operands[1])));
 DONE;
 })
 (define_split
 [(set (match_operand:SWI48 0 "register_operand")
 (define_insn "*anddi_2"
 [(set (reg FLAGS_REG)
 	(compare
 	 (and:DI
 	  (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
-	  (match_operand:DI 2 "x86_64_szext_general_operand" "Z,rem,re"))
+	  (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,m"))
 	 (const_int 0)))
-(set (match_operand:DI 0 "nonimmediate_operand" "=r,r,rm")
+(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r")
 	(and:DI (match_dup 1) (match_dup 2)))]
 "TARGET_64BIT
 && ix86_match_ccmode
 	(insn,
 	 /* If we are going to emit andl instead of andq, and the operands[2]
 (define_insn "*andqi_2_maybe_si"
 [(set (reg FLAGS_REG)
 	(compare (and:QI
 		  (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-		  (match_operand:QI 2 "general_operand" "qmn,qn,n"))
+		  (match_operand:QI 2 "general_operand" "qn,m,n"))
 		 (const_int 0)))
-(set (match_operand:QI 0 "nonimmediate_operand" "=q,qm,*r")
+(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,r")
 	(and:QI (match_dup 1) (match_dup 2)))]
 "ix86_binary_operator_ok (AND, QImode, operands)
 && ix86_match_ccmode (insn,
 			 CONST_INT_P (operands[2])
 			 && INTVAL (operands[2]) >= 0 ? CCNOmode : CCZmode)"
 return "and{l}\t{%2, %k0|%k0, %2}";
 }
 return "and{b}\t{%2, %0|%0, %2}";
 }
 [(set_attr "type" "alu")
-(set_attr "mode" "QI,QI,SI")])
+(set_attr "mode" "QI,QI,SI")
+;; Potential partial reg stall on alternative 2.
+(set (attr "preferred_for_speed")
+(cond [(eq_attr "alternative" "2")
+	      (symbol_ref "!TARGET_PARTIAL_REG_STALL")]
+	   (symbol_ref "true")))])
 (define_insn "*and<mode>_2"
 [(set (reg FLAGS_REG)
 	(compare (and:SWI124
 		  (match_operand:SWI124 1 "nonimmediate_operand" "%0,0")
-		  (match_operand:SWI124 2 "<general_operand>" "<g>,<r><i>"))
+		  (match_operand:SWI124 2 "<general_operand>" "<r><i>,m"))
 		 (const_int 0)))
-(set (match_operand:SWI124 0 "nonimmediate_operand" "=<r>,<r>m")
+(set (match_operand:SWI124 0 "nonimmediate_operand" "=<r>m,<r>")
 	(and:SWI124 (match_dup 1) (match_dup 2)))]
 "ix86_match_ccmode (insn, CCNOmode)
 && ix86_binary_operator_ok (AND, <MODE>mode, operands)"
 "and{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
-(define_insn "*andqi_2_slp"
-[(set (reg FLAGS_REG)
-	(compare (and:QI
-		   (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-		   (match_operand:QI 1 "nonimmediate_operand" "qmn,qn"))
-		 (const_int 0)))
-(set (strict_low_part (match_dup 0))
-	(and:QI (match_dup 0) (match_dup 1)))]
-"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& ix86_match_ccmode (insn, CCNOmode)
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-"and{b}\t{%1, %0|%0, %1}"
-[(set_attr "type" "alu1")
-(set_attr "mode" "QI")])
 (define_insn "andqi_ext_1"
 [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "+Q,Q")
 			 (const_int 8)
 			 (const_int 8))
 operands[1] = gen_lowpart (QImode, operands[1]);
 operands[2] = gen_int_mode (INTVAL (operands[2]), QImode);
 })
 (define_insn "*andndi3_doubleword"
-[(set (match_operand:DI 0 "register_operand" "=&r,r,r,&r")
+[(set (match_operand:DI 0 "register_operand")
 	(and:DI
-	  (not:DI (match_operand:DI 1 "register_operand" "r,0,r,0"))
+	  (not:DI (match_operand:DI 1 "register_operand"))
-	  (match_operand:DI 2 "nonimmediate_operand" "rm,rm,0,rm")))
+	  (match_operand:DI 2 "nonimmediate_operand")))
 (clobber (reg:CC FLAGS_REG))]
-"!TARGET_64BIT && TARGET_STV && TARGET_SSE2"
+"!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-"#"
+&& ix86_pre_reload_split ()"
-[(set_attr "isa" "bmi,bmi,bmi,*")])
+"#")
 (define_split
 [(set (match_operand:DI 0 "register_operand")
 	(and:DI
 	  (not:DI (match_operand:DI 1 "register_operand"))
 	  (match_operand:DI 2 "nonimmediate_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "!TARGET_64BIT && TARGET_BMI && TARGET_STV && TARGET_SSE2
-&& reload_completed"
+&& can_create_pseudo_p ()"
 [(parallel [(set (match_dup 0)
 		   (and:SI (not:SI (match_dup 1)) (match_dup 2)))
 	      (clobber (reg:CC FLAGS_REG))])
 (parallel [(set (match_dup 3)
 		   (and:SI (not:SI (match_dup 4)) (match_dup 5)))
 "split_double_mode (DImode, &operands[0], 3, &operands[0], &operands[3]);")
 (define_split
 [(set (match_operand:DI 0 "register_operand")
 	(and:DI
-	  (not:DI (match_dup 0))
+	  (not:DI (match_operand:DI 1 "register_operand"))
-	  (match_operand:DI 1 "nonimmediate_operand")))
+	  (match_operand:DI 2 "nonimmediate_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "!TARGET_64BIT && !TARGET_BMI && TARGET_STV && TARGET_SSE2
-&& reload_completed"
+&& can_create_pseudo_p ()"
-[(set (match_dup 0) (not:SI (match_dup 0)))
+[(set (match_dup 6) (not:SI (match_dup 1)))
 (parallel [(set (match_dup 0)
-		   (and:SI (match_dup 0) (match_dup 1)))
+		   (and:SI (match_dup 6) (match_dup 2)))
 	      (clobber (reg:CC FLAGS_REG))])
-(set (match_dup 2) (not:SI (match_dup 2)))
+(set (match_dup 7) (not:SI (match_dup 4)))
-(parallel [(set (match_dup 2)
+(parallel [(set (match_dup 3)
-		   (and:SI (match_dup 2) (match_dup 3)))
+		   (and:SI (match_dup 7) (match_dup 5)))
 	      (clobber (reg:CC FLAGS_REG))])]
-"split_double_mode (DImode, &operands[0], 2, &operands[0], &operands[2]);")
+{
+operands[6] = gen_reg_rtx (SImode);
+operands[7] = gen_reg_rtx (SImode);
+split_double_mode (DImode, &operands[0], 3, &operands[0], &operands[3]);
+})
 (define_insn "*andn<mode>_1"
 [(set (match_operand:SWI48 0 "register_operand" "=r,r")
 	(and:SWI48
 	  (not:SWI48 (match_operand:SWI48 1 "register_operand" "r,r"))
 ;; %%% This used to optimize known byte-wide and operations to memory.
 ;; If this is considered useful, it should be done with splitters.
 (define_expand "<code><mode>3"
-[(set (match_operand:SWIM1248x 0 "nonimmediate_operand")
+[(set (match_operand:SWIM1248s 0 "nonimmediate_operand")
-	(any_or:SWIM1248x (match_operand:SWIM1248x 1 "nonimmediate_operand")
+	(any_or:SWIM1248s (match_operand:SWIM1248s 1 "nonimmediate_operand")
-			     (match_operand:SWIM1248x 2 "<general_operand>")))]
+			  (match_operand:SWIM1248s 2 "<general_operand>")))]
 ""
 "ix86_expand_binary_operator (<CODE>, <MODE>mode, operands); DONE;")
 (define_insn_and_split "*<code>di3_doubleword"
-[(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r")
+[(set (match_operand:DI 0 "nonimmediate_operand")
 	(any_or:DI
-	 (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
+	 (match_operand:DI 1 "nonimmediate_operand")
-	 (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm")))
+	 (match_operand:DI 2 "x86_64_szext_general_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-&& ix86_binary_operator_ok (<CODE>, DImode, operands)"
+&& ix86_binary_operator_ok (<CODE>, DImode, operands)
+&& ix86_pre_reload_split ()"
 "#"
-"&& reload_completed"
+"&& 1"
 [(const_int 0)]
 {
 split_double_mode (DImode, &operands[0], 3, &operands[0], &operands[3]);
-if (operands[2] == constm1_rtx)
+if (operands[2] == const0_rtx)
+emit_move_insn (operands[0], operands[1]);
+else if (operands[2] == constm1_rtx)
 {
 if (<CODE> == IOR)
-	{
+	emit_move_insn (operands[0], constm1_rtx);
-	  operands[1] = constm1_rtx;
-	  ix86_expand_move (SImode, &operands[0]);
-	}
 else
 	ix86_expand_unary_operator (NOT, SImode, &operands[0]);
 }
-else if (operands[2] != const0_rtx)
+else
 ix86_expand_binary_operator (<CODE>, SImode, &operands[0]);
-else if (operands[5] == const0_rtx)
-emit_note (NOTE_INSN_DELETED);
+if (operands[5] == const0_rtx)
-if (operands[5] == constm1_rtx)
+emit_move_insn (operands[3], operands[4]);
+else if (operands[5] == constm1_rtx)
 {
 if (<CODE> == IOR)
-	{
+	emit_move_insn (operands[3], constm1_rtx);
-	  operands[4] = constm1_rtx;
-	  ix86_expand_move (SImode, &operands[3]);
-	}
 else
 	ix86_expand_unary_operator (NOT, SImode, &operands[3]);
 }
-else if (operands[5] != const0_rtx)
+else
 ix86_expand_binary_operator (<CODE>, SImode, &operands[3]);
 DONE;
 })
 (define_insn "*<code><mode>_1"
-[(set (match_operand:SWI248 0 "nonimmediate_operand" "=r,rm")
+[(set (match_operand:SWI248 0 "nonimmediate_operand" "=rm,r")
 	(any_or:SWI248
 	 (match_operand:SWI248 1 "nonimmediate_operand" "%0,0")
-	 (match_operand:SWI248 2 "<general_operand>" "<g>,r<i>")))
+	 (match_operand:SWI248 2 "<general_operand>" "r<i>,m")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 "<logic>{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
 "<logic>{l}\t{%2, %k0|%k0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "SI")])
 (define_insn "*<code>qi_1"
-[(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
+[(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,r")
 	(any_or:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-		   (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
+		   (match_operand:QI 2 "general_operand" "qn,m,rn")))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, QImode, operands)"
 "@
 <logic>{b}\t{%2, %0|%0, %2}
 <logic>{b}\t{%2, %0|%0, %2}
 (set (attr "preferred_for_speed")
 (cond [(eq_attr "alternative" "2")
 	      (symbol_ref "!TARGET_PARTIAL_REG_STALL")]
 	   (symbol_ref "true")))])
-(define_insn "*<code>qi_1_slp"
+(define_insn "*<code><mode>_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+q,m"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(any_or:QI (match_dup 0)
+	(any_or:SWI12 (match_operand:SWI12 1 "nonimmediate_operand" "%0")
-		   (match_operand:QI 1 "general_operand" "qmn,qn")))
+		      (match_operand:SWI12 2 "general_operand" "<r>mn")))
 (clobber (reg:CC FLAGS_REG))]
 "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-"<logic>{b}\t{%1, %0|%0, %1}"
+&& (rtx_equal_p (operands[0], operands[1])
-[(set_attr "type" "alu1")
+|| rtx_equal_p (operands[0], operands[2]))"
-(set_attr "mode" "QI")])
+"<logic>{<imodesuffix>}\t{%2, %0|%0, %2}"
+[(set_attr "type" "alu")
+(set_attr "mode" "<MODE>")])
 (define_insn "*<code><mode>_2"
 [(set (reg FLAGS_REG)
 	(compare (any_or:SWI
 		  (match_operand:SWI 1 "nonimmediate_operand" "%0,0")
-		  (match_operand:SWI 2 "<general_operand>" "<g>,<r><i>"))
+		  (match_operand:SWI 2 "<general_operand>" "<r><i>,m"))
 		 (const_int 0)))
-(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m")
+(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
 	(any_or:SWI (match_dup 1) (match_dup 2)))]
 "ix86_match_ccmode (insn, CCNOmode)
 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
 "<logic>{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
 && ix86_binary_operator_ok (<CODE>, SImode, operands)"
 "<logic>{l}\t{%2, %k0|%k0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "SI")])
-(define_insn "*<code>qi_2_slp"
-[(set (reg FLAGS_REG)
-	(compare (any_or:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-			    (match_operand:QI 1 "general_operand" "qmn,qn"))
-		 (const_int 0)))
-(set (strict_low_part (match_dup 0))
-	(any_or:QI (match_dup 0) (match_dup 1)))]
-"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-&& ix86_match_ccmode (insn, CCNOmode)
-&& !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-"<logic>{b}\t{%1, %0|%0, %1}"
-[(set_attr "type" "alu1")
-(set_attr "mode" "QI")])
 (define_insn "*<code><mode>_3"
 [(set (reg FLAGS_REG)
 	(compare (any_or:SWI
 		  (match_operand:SWI 1 "nonimmediate_operand" "%0")
 "ix86_unary_operator_ok (NEG, <MODE>mode, operands)"
 "neg{<imodesuffix>}\t%0"
 [(set_attr "type" "negnot")
 (set_attr "mode" "<MODE>")])
-;; Combine is quite creative about this pattern.
 (define_insn "*negsi2_1_zext"
 [(set (match_operand:DI 0 "register_operand" "=r")
-	(lshiftrt:DI
+	(zero_extend:DI
-	  (neg:DI (ashift:DI (match_operand:DI 1 "register_operand" "0")
+	  (neg:SI (match_operand:SI 1 "register_operand" "0"))))
-			     (const_int 32)))
-	(const_int 32)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT && ix86_unary_operator_ok (NEG, SImode, operands)"
 "neg{l}\t%k0"
 [(set_attr "type" "negnot")
 (set_attr "mode" "SI")])
 (set_attr "mode" "<MODE>")])
 (define_insn "*negsi2_cmpz_zext"
 [(set (reg:CCZ FLAGS_REG)
 	(compare:CCZ
-	  (lshiftrt:DI
+	  (neg:SI (match_operand:SI 1 "register_operand" "0"))
-	    (neg:DI (ashift:DI
-		      (match_operand:DI 1 "register_operand" "0")
-		      (const_int 32)))
-	    (const_int 32))
 	  (const_int 0)))
 (set (match_operand:DI 0 "register_operand" "=r")
-	(lshiftrt:DI (neg:DI (ashift:DI (match_dup 1)
+	(zero_extend:DI
-					(const_int 32)))
+	  (neg:SI (match_dup 1))))]
-		     (const_int 32)))]
 "TARGET_64BIT && ix86_unary_operator_ok (NEG, SImode, operands)"
 "neg{l}\t%k0"
 [(set_attr "type" "negnot")
 (set_attr "mode" "SI")])
 && mode_signbit_p (<MODE>mode, operands[2])"
 "neg{<imodesuffix>}\t%0"
 [(set_attr "type" "negnot")
 (set_attr "mode" "<MODE>")])
-;; Changing of sign for FP values is doable using integer unit too.
+(define_expand "<code>tf2"
+[(set (match_operand:TF 0 "register_operand")
+	(absneg:TF (match_operand:TF 1 "register_operand")))]
+"TARGET_SSE"
+"ix86_expand_fp_absneg_operator (<CODE>, TFmode, operands); DONE;")
+(define_insn "*<code>tf2_1"
+[(set (match_operand:TF 0 "register_operand" "=x,x,Yv,Yv")
+	(absneg:TF
+	  (match_operand:TF 1 "vector_operand" "0,xBm,Yv,m")))
+(use (match_operand:TF 2 "vector_operand" "xBm,0,Yvm,Yv"))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_SSE"
+"#"
+[(set_attr "isa" "noavx,noavx,avx,avx")])
+(define_insn "*nabstf2_1"
+[(set (match_operand:TF 0 "register_operand" "=x,x,Yv,Yv")
+	(neg:TF
+	  (abs:TF
+	    (match_operand:TF 1 "vector_operand" "0,xBm,Yv,m"))))
+(use (match_operand:TF 2 "vector_operand" "xBm,0,Yvm,Yv"))]
+"TARGET_SSE"
+"#"
+[(set_attr "isa" "noavx,noavx,avx,avx")])
+;; Special expand pattern to handle integer mode abs
+(define_expand "abs<mode>2"
+[(set (match_operand:SWI48x 0 "register_operand")
+(abs:SWI48x
+(match_operand:SWI48x 1 "register_operand")))]
+"TARGET_EXPAND_ABS"
+{
+machine_mode mode = <MODE>mode;
+/* Generate rtx abs using abs (x) = (((signed) x >> (W-1)) ^ x) -
+((signed) x >> (W-1)) */
+rtx shift_amount = gen_int_mode (GET_MODE_PRECISION (mode) - 1, QImode);
+rtx shift_dst = expand_simple_binop (mode, ASHIFTRT, operands[1],
+					 shift_amount, NULL_RTX,
+					 0, OPTAB_DIRECT);
+rtx xor_dst = expand_simple_binop (mode, XOR, shift_dst, operands[1],
+				       operands[0], 0, OPTAB_DIRECT);
+rtx minus_dst = expand_simple_binop (mode, MINUS, xor_dst, shift_dst,
+					 operands[0], 0, OPTAB_DIRECT);
+if (!rtx_equal_p (minus_dst, operands[0]))
+emit_move_insn (operands[0], minus_dst);
+DONE;
+})
 (define_expand "<code><mode>2"
 [(set (match_operand:X87MODEF 0 "register_operand")
 	(absneg:X87MODEF (match_operand:X87MODEF 1 "register_operand")))]
 "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
-(define_insn "*absneg<mode>2"
+;; Changing of sign for FP values is doable using integer unit too.
-[(set (match_operand:MODEF 0 "register_operand" "=Yv,Yv,f,!r")
+(define_insn "*<code><mode>2_i387_1"
-	(match_operator:MODEF 3 "absneg_operator"
+[(set (match_operand:X87MODEF 0 "register_operand" "=f,!r")
-	  [(match_operand:MODEF 1 "register_operand" "0,Yv,0,0")]))
+	(absneg:X87MODEF
-(use (match_operand:<ssevecmode> 2 "nonimmediate_operand" "Yvm,0,X,X"))
+	  (match_operand:X87MODEF 1 "register_operand" "0,0")))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_80387"
+"#")
+(define_split
+[(set (match_operand:X87MODEF 0 "fp_register_operand")
+	(absneg:X87MODEF (match_operand:X87MODEF 1 "fp_register_operand")))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_80387 && reload_completed"
+[(set (match_dup 0) (absneg:X87MODEF (match_dup 1)))])
+(define_split
+[(set (match_operand:X87MODEF 0 "general_reg_operand")
+	(absneg:X87MODEF (match_operand:X87MODEF 1 "general_reg_operand")))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_80387 && reload_completed"
+[(const_int 0)]
+"ix86_split_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
+(define_insn "*<code><mode>2_1"
+[(set (match_operand:MODEF 0 "register_operand" "=x,x,Yv,f,!r")
+	(absneg:MODEF
+	  (match_operand:MODEF 1 "register_operand" "0,x,Yv,0,0")))
+(use (match_operand:<ssevecmode> 2 "vector_operand" "xBm,0,Yvm,X,X"))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
 "#"
-[(set (attr "enabled")
+[(set_attr "isa" "noavx,noavx,avx,*,*")
+(set (attr "enabled")
 (if_then_else
 (match_test ("SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"))
 (if_then_else
-	 (eq_attr "alternative" "2")
+	 (eq_attr "alternative" "3,4")
 	 (symbol_ref "TARGET_MIX_SSE_I387")
-	 (symbol_ref "true"))
+	 (const_string "*"))
 (if_then_else
-	 (eq_attr "alternative" "2,3")
+	 (eq_attr "alternative" "3,4")
 	 (symbol_ref "true")
 	 (symbol_ref "false"))))])
-(define_insn "*absnegxf2_i387"
-[(set (match_operand:XF 0 "register_operand" "=f,!r")
-	(match_operator:XF 3 "absneg_operator"
-	  [(match_operand:XF 1 "register_operand" "0,0")]))
-(use (match_operand 2))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_80387"
-"#")
-(define_expand "<code>tf2"
-[(set (match_operand:TF 0 "register_operand")
-	(absneg:TF (match_operand:TF 1 "register_operand")))]
-"TARGET_SSE"
-"ix86_expand_fp_absneg_operator (<CODE>, TFmode, operands); DONE;")
-(define_insn "*absnegtf2_sse"
-[(set (match_operand:TF 0 "register_operand" "=Yv,Yv")
-	(match_operator:TF 3 "absneg_operator"
-	  [(match_operand:TF 1 "register_operand" "0,Yv")]))
-(use (match_operand:TF 2 "nonimmediate_operand" "Yvm,0"))
-(clobber (reg:CC FLAGS_REG))]
-"TARGET_SSE"
-"#")
-;; Splitters for fp abs and neg.
 (define_split
-[(set (match_operand 0 "fp_register_operand")
+[(set (match_operand:SSEMODEF 0 "sse_reg_operand")
-	(match_operator 1 "absneg_operator" [(match_dup 0)]))
+	(absneg:SSEMODEF
-(use (match_operand 2))
+	  (match_operand:SSEMODEF 1 "vector_operand")))
-(clobber (reg:CC FLAGS_REG))]
+(use (match_operand:<ssevecmodef> 2 "vector_operand"))
-"reload_completed"
+(clobber (reg:CC FLAGS_REG))]
-[(set (match_dup 0) (match_op_dup 1 [(match_dup 0)]))])
+"((SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| (TARGET_SSE && (<MODE>mode == TFmode)))
-(define_split
+&& reload_completed"
-[(set (match_operand 0 "sse_reg_operand")
-	(match_operator 3 "absneg_operator"
-	  [(match_operand 1 "register_operand")]))
-(use (match_operand 2 "nonimmediate_operand"))
-(clobber (reg:CC FLAGS_REG))]
-"reload_completed"
 [(set (match_dup 0) (match_dup 3))]
 {
-machine_mode mode = GET_MODE (operands[0]);
+machine_mode mode = <MODE>mode;
-machine_mode vmode = GET_MODE (operands[2]);
+machine_mode vmode = <ssevecmodef>mode;
-rtx tmp;
+enum rtx_code absneg_op = <CODE> == ABS ? AND : XOR;
 operands[0] = lowpart_subreg (vmode, operands[0], mode);
 operands[1] = lowpart_subreg (vmode, operands[1], mode);
-if (operands_match_p (operands[0], operands[2]))
-std::swap (operands[1], operands[2]);
+if (TARGET_AVX)
-if (GET_CODE (operands[3]) == ABS)
-tmp = gen_rtx_AND (vmode, operands[1], operands[2]);
-else
-tmp = gen_rtx_XOR (vmode, operands[1], operands[2]);
-operands[3] = tmp;
-})
-(define_split
-[(set (match_operand:SF 0 "general_reg_operand")
-	(match_operator:SF 1 "absneg_operator" [(match_dup 0)]))
-(use (match_operand:V4SF 2))
-(clobber (reg:CC FLAGS_REG))]
-"reload_completed"
-[(parallel [(set (match_dup 0) (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-rtx tmp;
-operands[0] = gen_lowpart (SImode, operands[0]);
-if (GET_CODE (operands[1]) == ABS)
 {
-tmp = gen_int_mode (0x7fffffff, SImode);
+if (MEM_P (operands[1]))
-tmp = gen_rtx_AND (SImode, operands[0], tmp);
+std::swap (operands[1], operands[2]);
 }
 else
+{
+if (operands_match_p (operands[0], operands[2]))
+std::swap (operands[1], operands[2]);
+}
+operands[3]
+= gen_rtx_fmt_ee (absneg_op, vmode, operands[1], operands[2]);
+})
+(define_split
+[(set (match_operand:MODEF 0 "fp_register_operand")
+	(absneg:MODEF (match_operand:MODEF 1 "fp_register_operand")))
+(use (match_operand 2))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_80387 && reload_completed"
+[(set (match_dup 0) (absneg:MODEF (match_dup 1)))])
+(define_split
+[(set (match_operand:MODEF 0 "general_reg_operand")
+	(absneg:MODEF (match_operand:MODEF 1 "general_reg_operand")))
+(use (match_operand 2))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_80387 && reload_completed"
+[(const_int 0)]
+"ix86_split_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
+(define_insn "*nabs<mode>2_1"
+[(set (match_operand:MODEF 0 "register_operand" "=x,x,Yv")
+	(neg:MODEF
+	  (abs:MODEF
+	    (match_operand:MODEF 1 "register_operand" "0,x,Yv"))))
+(use (match_operand:<ssevecmode> 2 "vector_operand" "xBm,0,Yvm"))]
+"SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+"#"
+[(set_attr "isa" "noavx,noavx,avx")])
+(define_split
+[(set (match_operand:SSEMODEF 0 "sse_reg_operand")
+	(neg:SSEMODEF
+	  (abs:SSEMODEF
+	    (match_operand:SSEMODEF 1 "vector_operand"))))
+(use (match_operand:<ssevecmodef> 2 "vector_operand"))]
+"((SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| (TARGET_SSE && (<MODE>mode == TFmode)))
+&& reload_completed"
+[(set (match_dup 0) (match_dup 3))]
+{
+machine_mode mode = <MODE>mode;
+machine_mode vmode = <ssevecmodef>mode;
+operands[0] = lowpart_subreg (vmode, operands[0], mode);
+operands[1] = lowpart_subreg (vmode, operands[1], mode);
+if (TARGET_AVX)
 {
-tmp = gen_int_mode (0x80000000, SImode);
+if (MEM_P (operands[1]))
-tmp = gen_rtx_XOR (SImode, operands[0], tmp);
+std::swap (operands[1], operands[2]);
-}
-operands[1] = tmp;
-})
-(define_split
-[(set (match_operand:DF 0 "general_reg_operand")
-	(match_operator:DF 1 "absneg_operator" [(match_dup 0)]))
-(use (match_operand 2))
-(clobber (reg:CC FLAGS_REG))]
-"reload_completed"
-[(parallel [(set (match_dup 0) (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-rtx tmp;
-if (TARGET_64BIT)
-{
-tmp = gen_lowpart (DImode, operands[0]);
-tmp = gen_rtx_ZERO_EXTRACT (DImode, tmp, const1_rtx, GEN_INT (63));
-operands[0] = tmp;
-if (GET_CODE (operands[1]) == ABS)
-	tmp = const0_rtx;
-else
-	tmp = gen_rtx_NOT (DImode, tmp);
 }
 else
 {
-operands[0] = gen_highpart (SImode, operands[0]);
+if (operands_match_p (operands[0], operands[2]))
-if (GET_CODE (operands[1]) == ABS)
+std::swap (operands[1], operands[2]);
-	{
+}
-	  tmp = gen_int_mode (0x7fffffff, SImode);
-	  tmp = gen_rtx_AND (SImode, operands[0], tmp);
+operands[3]
-	}
+= gen_rtx_fmt_ee (IOR, vmode, operands[1], operands[2]);
-else
-	{
-	  tmp = gen_int_mode (0x80000000, SImode);
-	  tmp = gen_rtx_XOR (SImode, operands[0], tmp);
-	}
-}
-operands[1] = tmp;
-})
-(define_split
-[(set (match_operand:XF 0 "general_reg_operand")
-	(match_operator:XF 1 "absneg_operator" [(match_dup 0)]))
-(use (match_operand 2))
-(clobber (reg:CC FLAGS_REG))]
-"reload_completed"
-[(parallel [(set (match_dup 0) (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-rtx tmp;
-operands[0] = gen_rtx_REG (SImode,
-			     REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
-if (GET_CODE (operands[1]) == ABS)
-{
-tmp = GEN_INT (0x7fff);
-tmp = gen_rtx_AND (SImode, operands[0], tmp);
-}
-else
-{
-tmp = GEN_INT (0x8000);
-tmp = gen_rtx_XOR (SImode, operands[0], tmp);
-}
-operands[1] = tmp;
 })
 ;; Conditionalize these after reload. If they match before reload, we
 ;; lose the clobber and ability to use integer instructions.
-(define_insn "*<code><mode>2_1"
+(define_insn "*<code><mode>2_i387"
 [(set (match_operand:X87MODEF 0 "register_operand" "=f")
 	(absneg:X87MODEF (match_operand:X87MODEF 1 "register_operand" "0")))]
-"TARGET_80387
+"TARGET_80387 && reload_completed"
-&& (reload_completed
-|| !(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH))"
 "<absneg_mnemonic>"
 [(set_attr "type" "fsgn")
 (set_attr "mode" "<MODE>")])
 ;; Copysign instructions
-(define_mode_iterator CSGNMODE [SF DF TF])
-(define_mode_attr CSGNVMODE [(SF "V4SF") (DF "V2DF") (TF "TF")])
 (define_expand "copysign<mode>3"
-[(match_operand:CSGNMODE 0 "register_operand")
+[(match_operand:SSEMODEF 0 "register_operand")
-(match_operand:CSGNMODE 1 "nonmemory_operand")
+(match_operand:SSEMODEF 1 "nonmemory_operand")
-(match_operand:CSGNMODE 2 "register_operand")]
+(match_operand:SSEMODEF 2 "register_operand")]
 "(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 || (TARGET_SSE && (<MODE>mode == TFmode))"
 "ix86_expand_copysign (operands); DONE;")
-(define_insn_and_split "copysign<mode>3_const"
+(define_insn_and_split "@copysign<mode>3_const"
-[(set (match_operand:CSGNMODE 0 "register_operand" "=Yv")
+[(set (match_operand:SSEMODEF 0 "register_operand" "=Yv")
-	(unspec:CSGNMODE
+	(unspec:SSEMODEF
-	  [(match_operand:<CSGNVMODE> 1 "nonimm_or_0_operand" "YvmC")
+	  [(match_operand:<ssevecmodef> 1 "nonimm_or_0_operand" "YvmC")
-	   (match_operand:CSGNMODE 2 "register_operand" "0")
+	   (match_operand:SSEMODEF 2 "register_operand" "0")
-	   (match_operand:<CSGNVMODE> 3 "nonimmediate_operand" "Yvm")]
+	   (match_operand:<ssevecmodef> 3 "nonimmediate_operand" "Yvm")]
 	  UNSPEC_COPYSIGN))]
 "(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 || (TARGET_SSE && (<MODE>mode == TFmode))"
 "#"
 "&& reload_completed"
 [(const_int 0)]
 "ix86_split_copysign_const (operands); DONE;")
-(define_insn "copysign<mode>3_var"
+(define_insn "@copysign<mode>3_var"
-[(set (match_operand:CSGNMODE 0 "register_operand" "=Yv,Yv,Yv,Yv,Yv")
+[(set (match_operand:SSEMODEF 0 "register_operand" "=Yv,Yv,Yv,Yv,Yv")
-	(unspec:CSGNMODE
+	(unspec:SSEMODEF
-	  [(match_operand:CSGNMODE 2 "register_operand"	"Yv,0,0,Yv,Yv")
+	  [(match_operand:SSEMODEF 2 "register_operand"	"Yv,0,0,Yv,Yv")
-	   (match_operand:CSGNMODE 3 "register_operand"	"1,1,Yv,1,Yv")
+	   (match_operand:SSEMODEF 3 "register_operand"	"1,1,Yv,1,Yv")
-	   (match_operand:<CSGNVMODE> 4
+	   (match_operand:<ssevecmodef> 4
 	     "nonimmediate_operand" "X,Yvm,Yvm,0,0")
-	   (match_operand:<CSGNVMODE> 5
+	   (match_operand:<ssevecmodef> 5
 	     "nonimmediate_operand" "0,Yvm,1,Yvm,1")]
 	  UNSPEC_COPYSIGN))
-(clobber (match_scratch:<CSGNVMODE> 1 "=Yv,Yv,Yv,Yv,Yv"))]
+(clobber (match_scratch:<ssevecmodef> 1 "=Yv,Yv,Yv,Yv,Yv"))]
 "(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 || (TARGET_SSE && (<MODE>mode == TFmode))"
 "#")
 (define_split
-[(set (match_operand:CSGNMODE 0 "register_operand")
+[(set (match_operand:SSEMODEF 0 "register_operand")
-	(unspec:CSGNMODE
+	(unspec:SSEMODEF
-	  [(match_operand:CSGNMODE 2 "register_operand")
+	  [(match_operand:SSEMODEF 2 "register_operand")
-	   (match_operand:CSGNMODE 3 "register_operand")
+	   (match_operand:SSEMODEF 3 "register_operand")
-	   (match_operand:<CSGNVMODE> 4)
+	   (match_operand:<ssevecmodef> 4)
-	   (match_operand:<CSGNVMODE> 5)]
+	   (match_operand:<ssevecmodef> 5)]
 	  UNSPEC_COPYSIGN))
-(clobber (match_scratch:<CSGNVMODE> 1))]
+(clobber (match_scratch:<ssevecmodef> 1))]
 "((SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 || (TARGET_SSE && (<MODE>mode == TFmode)))
 && reload_completed"
 [(const_int 0)]
 "ix86_split_copysign_var (operands); DONE;")
+(define_expand "xorsign<mode>3"
+[(match_operand:MODEF 0 "register_operand")
+(match_operand:MODEF 1 "register_operand")
+(match_operand:MODEF 2 "register_operand")]
+"SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+"ix86_expand_xorsign (operands); DONE;")
+(define_insn_and_split "@xorsign<mode>3_1"
+[(set (match_operand:MODEF 0 "register_operand" "=Yv")
+	(unspec:MODEF
+	  [(match_operand:MODEF 1 "register_operand" "Yv")
+	   (match_operand:MODEF 2 "register_operand" "0")
+	   (match_operand:<ssevecmode> 3 "nonimmediate_operand" "Yvm")]
+	  UNSPEC_XORSIGN))]
+"SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+"#"
+"&& reload_completed"
+[(const_int 0)]
+"ix86_split_xorsign (operands); DONE;")
 ;; One complement instructions
 (define_expand "one_cmpl<mode>2"
-[(set (match_operand:SWIM1248x 0 "nonimmediate_operand")
+[(set (match_operand:SWIM1248s 0 "nonimmediate_operand")
-	(not:SWIM1248x (match_operand:SWIM1248x 1 "nonimmediate_operand")))]
+	(not:SWIM1248s (match_operand:SWIM1248s 1 "nonimmediate_operand")))]
 ""
 "ix86_expand_unary_operator (NOT, <MODE>mode, operands); DONE;")
 (define_insn_and_split "*one_cmpldi2_doubleword"
-[(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
+[(set (match_operand:DI 0 "nonimmediate_operand")
-	(not:DI (match_operand:DI 1 "nonimmediate_operand" "0")))]
+	(not:DI (match_operand:DI 1 "nonimmediate_operand")))]
 "!TARGET_64BIT && TARGET_STV && TARGET_SSE2
-&& ix86_unary_operator_ok (NOT, DImode, operands)"
+&& ix86_unary_operator_ok (NOT, DImode, operands)
+&& ix86_pre_reload_split ()"
 "#"
-"&& reload_completed"
+"&& 1"
 [(set (match_dup 0)
 	(not:SI (match_dup 1)))
 (set (match_dup 2)
 	(not:SI (match_dup 3)))]
 "split_double_mode (DImode, &operands[0], 2, &operands[0], &operands[2]);")
 "ix86_unary_operator_ok (NOT, <MODE>mode, operands)"
 "not{<imodesuffix>}\t%0"
 [(set_attr "type" "negnot")
 (set_attr "mode" "<MODE>")])
-;; ??? Currently never generated - xor is used instead.
 (define_insn "*one_cmplsi2_1_zext"
 [(set (match_operand:DI 0 "register_operand" "=r")
 	(zero_extend:DI
 	  (not:SI (match_operand:SI 1 "register_operand" "0"))))]
 "TARGET_64BIT && ix86_unary_operator_ok (NOT, SImode, operands)"
 		   (match_op_dup 2 [(xor:SWI (match_dup 3) (const_int -1))
 				    (const_int 0)]))
 	      (set (match_dup 1)
 		   (xor:SWI (match_dup 3) (const_int -1)))])])
-;; ??? Currently never generated - xor is used instead.
 (define_insn "*one_cmplsi2_2_zext"
 [(set (reg FLAGS_REG)
 	(compare (not:SI (match_operand:SI 1 "register_operand" "0"))
 		 (const_int 0)))
 (set (match_operand:DI 0 "register_operand" "=r")
 	    (and:SI
 	      (match_operand:SI 2 "register_operand" "c")
 	      (match_operand:SI 3 "const_int_operand")) 0)))
 (clobber (reg:CC FLAGS_REG))]
 "(INTVAL (operands[3]) & (<MODE_SIZE> * BITS_PER_UNIT)) == 0
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 6)
 	   (ior:DWIH (ashift:DWIH (match_dup 6) (match_dup 2))
 	  (and:QI
 	    (match_operand:QI 2 "register_operand" "c")
 	    (match_operand:QI 3 "const_int_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "(INTVAL (operands[3]) & (<MODE_SIZE> * BITS_PER_UNIT)) == 0
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 6)
 	   (ior:DWIH (ashift:DWIH (match_dup 6) (match_dup 2))
 (define_split
 [(set (match_operand:DWI 0 "register_operand")
 	(ashift:DWI (match_operand:DWI 1 "nonmemory_operand")
 		    (match_operand:QI 2 "nonmemory_operand")))
 (clobber (reg:CC FLAGS_REG))]
-"(optimize && flag_peephole2) ? epilogue_completed : reload_completed"
+"epilogue_completed"
 [(const_int 0)]
 "ix86_split_ashl (operands, NULL_RTX, <MODE>mode); DONE;")
 ;; By default we don't ask for a scratch register, because when DWImode
 ;; values are manipulated, registers are already at a premium.  But if
 (set_attr "pent_pair" "np")
 (set_attr "athlon_decode" "vector")
 (set_attr "amdfam10_decode" "vector")
 (set_attr "bdver1_decode" "vector")])
-(define_expand "x86_shift<mode>_adj_1"
+(define_expand "@x86_shift<mode>_adj_1"
 [(set (reg:CCZ FLAGS_REG)
 	(compare:CCZ (and:QI (match_operand:QI 2 "register_operand")
 			     (match_dup 4))
 		     (const_int 0)))
 (set (match_operand:SWI48 0 "register_operand")
 			    (match_operand:SWI48 3 "register_operand")
 			    (match_dup 1)))]
 "TARGET_CMOVE"
 "operands[4] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));")
-(define_expand "x86_shift<mode>_adj_2"
+(define_expand "@x86_shift<mode>_adj_2"
 [(use (match_operand:SWI48 0 "register_operand"))
 (use (match_operand:SWI48 1 "register_operand"))
 (use (match_operand:QI 2 "register_operand"))]
 ""
 {
 	      (match_operand:SI 3 "const_int_operand")) 0)))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (ASHIFT, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (ashift:SWI48 (match_dup 1)
 	    (match_operand:QI 3 "const_int_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (ASHIFT, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (ashift:SWI48 (match_dup 1)
 (set (attr "preferred_for_speed")
 (cond [(eq_attr "alternative" "1")
 	      (symbol_ref "!TARGET_PARTIAL_REG_STALL")]
 	   (symbol_ref "true")))])
-(define_insn "*ashlqi3_1_slp"
+(define_insn "*ashl<mode>3_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(ashift:QI (match_dup 0)
+	(ashift:SWI12 (match_operand:SWI12 1 "register_operand" "0")
-		   (match_operand:QI 1 "nonmemory_operand" "cI")))
+		      (match_operand:QI 2 "nonmemory_operand" "cI")))
 (clobber (reg:CC FLAGS_REG))]
-"(optimize_function_for_size_p (cfun)
+"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-|| !TARGET_PARTIAL_FLAG_REG_STALL
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-|| (operands[1] == const1_rtx
+&& rtx_equal_p (operands[0], operands[1])"
-	&& (TARGET_SHIFT1
-	    || (TARGET_DOUBLE_WITH_ADD && REG_P (operands[0])))))"
 {
 switch (get_attr_type (insn))
 {
-case TYPE_ALU1:
+case TYPE_ALU:
-gcc_assert (operands[1] == const1_rtx);
+gcc_assert (operands[2] == const1_rtx);
-return "add{b}\t%0, %0";
+return "add{<imodesuffix>}\t%0, %0";
 default:
-if (operands[1] == const1_rtx
+if (operands[2] == const1_rtx
 	  && (TARGET_SHIFT1 || optimize_function_for_size_p (cfun)))
-	return "sal{b}\t%0";
+	return "sal{<imodesuffix>}\t%0";
 else
-	return "sal{b}\t{%1, %0|%0, %1}";
+	return "sal{<imodesuffix>}\t{%2, %0|%0, %2}";
 }
 }
 [(set (attr "type")
-(cond [(and (and (match_test "TARGET_DOUBLE_WITH_ADD")
+(cond [(and (match_test "TARGET_DOUBLE_WITH_ADD")
-		      (match_operand 0 "register_operand"))
+		 (match_operand 2 "const1_operand"))
-		 (match_operand 1 "const1_operand"))
+	      (const_string "alu")
-	      (const_string "alu1")
 	   ]
-	   (const_string "ishift1")))
+	   (const_string "ishift")))
 (set (attr "length_immediate")
 (if_then_else
-(ior (eq_attr "type" "alu1")
+(ior (eq_attr "type" "alu")
-	    (and (eq_attr "type" "ishift1")
+	    (and (eq_attr "type" "ishift")
-		 (and (match_operand 1 "const1_operand")
+		 (and (match_operand 2 "const1_operand")
 		      (ior (match_test "TARGET_SHIFT1")
 			   (match_test "optimize_function_for_size_p (cfun)")))))
 (const_string "0")
 (const_string "*")))
-(set_attr "mode" "QI")])
+(set_attr "mode" "<MODE>")])
 ;; Convert ashift to the lea pattern to avoid flags dependency.
 (define_split
 [(set (match_operand:SWI 0 "register_operand")
 	(ashift:SWI (match_operand:SWI 1 "index_register_operand")
 	      (match_operand:SI 3 "const_int_operand")) 0)))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_shiftrt:SWI48 (match_dup 1)
 	    (match_operand:QI 3 "const_int_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_shiftrt:SWI48 (match_dup 1)
 	    (and:SI
 	      (match_operand:SI 2 "register_operand" "c")
 	      (match_operand:SI 3 "const_int_operand")) 0)))
 (clobber (reg:CC FLAGS_REG))]
 "(INTVAL (operands[3]) & (<MODE_SIZE> * BITS_PER_UNIT)) == 0
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 4)
 	   (ior:DWIH (lshiftrt:DWIH (match_dup 4) (match_dup 2))
 	  (and:QI
 	    (match_operand:QI 2 "register_operand" "c")
 	    (match_operand:QI 3 "const_int_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "(INTVAL (operands[3]) & (<MODE_SIZE> * BITS_PER_UNIT)) == 0
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 4)
 	   (ior:DWIH (lshiftrt:DWIH (match_dup 4) (match_dup 2))
 	(any_shiftrt:DWI (match_operand:DWI 1 "register_operand" "0")
 			 (match_operand:QI 2 "nonmemory_operand" "<S>c")))
 (clobber (reg:CC FLAGS_REG))]
 ""
 "#"
-"(optimize && flag_peephole2) ? epilogue_completed : reload_completed"
+"epilogue_completed"
 [(const_int 0)]
 "ix86_split_<shift_insn> (operands, NULL_RTX, <MODE>mode); DONE;"
 [(set_attr "type" "multi")])
 ;; By default we don't ask for a scratch register, because when DWImode
 (set_attr "pent_pair" "np")
 (set_attr "athlon_decode" "vector")
 (set_attr "amdfam10_decode" "vector")
 (set_attr "bdver1_decode" "vector")])
-(define_insn "ashrdi3_cvt"
+;; Base name for insn mnemonic.
-[(set (match_operand:DI 0 "nonimmediate_operand" "=*d,rm")
+(define_mode_attr cvt_mnemonic
-	(ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "*a,0")
+[(SI "{cltd|cdq}") (DI "{cqto|cqo}")])
-		     (match_operand:QI 2 "const_int_operand")))
-(clobber (reg:CC FLAGS_REG))]
+(define_insn "ashr<mode>3_cvt"
-"TARGET_64BIT && INTVAL (operands[2]) == 63
+[(set (match_operand:SWI48 0 "nonimmediate_operand" "=*d,rm")
+	(ashiftrt:SWI48
+	  (match_operand:SWI48 1 "nonimmediate_operand" "*a,0")
+	  (match_operand:QI 2 "const_int_operand")))
+(clobber (reg:CC FLAGS_REG))]
+"INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)-1
 && (TARGET_USE_CLTD || optimize_function_for_size_p (cfun))
-&& ix86_binary_operator_ok (ASHIFTRT, DImode, operands)"
+&& ix86_binary_operator_ok (ASHIFTRT, <MODE>mode, operands)"
 "@
-{cqto|cqo}
+<cvt_mnemonic>
-sar{q}\t{%2, %0|%0, %2}"
+sar{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "imovx,ishift")
 (set_attr "prefix_0f" "0,*")
 (set_attr "length_immediate" "0,*")
 (set_attr "modrm" "0,1")
-(set_attr "mode" "DI")])
+(set_attr "mode" "<MODE>")])
 (define_insn "*ashrsi3_cvt_zext"
 [(set (match_operand:DI 0 "register_operand" "=*d,r")
 	(zero_extend:DI
 	  (ashiftrt:SI (match_operand:SI 1 "register_operand" "*a,0")
 (set_attr "prefix_0f" "0,*")
 (set_attr "length_immediate" "0,*")
 (set_attr "modrm" "0,1")
 (set_attr "mode" "SI")])
-(define_insn "ashrsi3_cvt"
+(define_expand "@x86_shift<mode>_adj_3"
-[(set (match_operand:SI 0 "nonimmediate_operand" "=*d,rm")
-	(ashiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "*a,0")
-		     (match_operand:QI 2 "const_int_operand")))
-(clobber (reg:CC FLAGS_REG))]
-"INTVAL (operands[2]) == 31
-&& (TARGET_USE_CLTD || optimize_function_for_size_p (cfun))
-&& ix86_binary_operator_ok (ASHIFTRT, SImode, operands)"
-"@
-{cltd|cdq}
-sar{l}\t{%2, %0|%0, %2}"
-[(set_attr "type" "imovx,ishift")
-(set_attr "prefix_0f" "0,*")
-(set_attr "length_immediate" "0,*")
-(set_attr "modrm" "0,1")
-(set_attr "mode" "SI")])
-(define_expand "x86_shift<mode>_adj_3"
 [(use (match_operand:SWI48 0 "register_operand"))
 (use (match_operand:SWI48 1 "register_operand"))
 (use (match_operand:QI 2 "register_operand"))]
 ""
 {
 		 (match_test "optimize_function_for_size_p (cfun)")))
 (const_string "0")
 (const_string "*")))
 (set_attr "mode" "<MODE>")])
-(define_insn "*<shift_insn>qi3_1_slp"
+(define_insn "*<shift_insn><mode>3_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(any_shiftrt:QI (match_dup 0)
+	(any_shiftrt:SWI12 (match_operand:SWI12 1 "register_operand" "0")
-			(match_operand:QI 1 "nonmemory_operand" "cI")))
+			   (match_operand:QI 2 "nonmemory_operand" "cI")))
 (clobber (reg:CC FLAGS_REG))]
-"(optimize_function_for_size_p (cfun)
+"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-|| !TARGET_PARTIAL_REG_STALL
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-|| (operands[1] == const1_rtx
+&& rtx_equal_p (operands[0], operands[1])"
-	&& TARGET_SHIFT1))"
+{
-{
+if (operands[2] == const1_rtx
-if (operands[1] == const1_rtx
 && (TARGET_SHIFT1 || optimize_function_for_size_p (cfun)))
-return "<shift>{b}\t%0";
+return "<shift>{<imodesuffix>}\t%0";
 else
-return "<shift>{b}\t{%1, %0|%0, %1}";
+return "<shift>{<imodesuffix>}\t{%2, %0|%0, %2}";
 }
-[(set_attr "type" "ishift1")
+[(set_attr "type" "ishift")
 (set (attr "length_immediate")
 (if_then_else
-(and (match_operand 1 "const1_operand")
+(and (match_operand 2 "const1_operand")
 	    (ior (match_test "TARGET_SHIFT1")
 		 (match_test "optimize_function_for_size_p (cfun)")))
 (const_string "0")
 (const_string "*")))
-(set_attr "mode" "QI")])
+(set_attr "mode" "<MODE>")])
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
 (define_insn "*<shift_insn><mode>3_cmp"
 	      (match_operand:SI 3 "const_int_operand")) 0)))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_rotate:SWI48 (match_dup 1)
 	    (match_operand:QI 3 "const_int_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_rotate:SWI48 (match_dup 1)
 		 (match_test "optimize_function_for_size_p (cfun)")))
 (const_string "0")
 (const_string "*")))
 (set_attr "mode" "<MODE>")])
-(define_insn "*<rotate_insn>qi3_1_slp"
+(define_insn "*<rotate_insn><mode>3_1_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
+[(set (strict_low_part (match_operand:SWI12 0 "register_operand" "+<r>"))
-	(any_rotate:QI (match_dup 0)
+	(any_rotate:SWI12 (match_operand:SWI12 1 "register_operand" "0")
-		       (match_operand:QI 1 "nonmemory_operand" "cI")))
+			  (match_operand:QI 2 "nonmemory_operand" "cI")))
 (clobber (reg:CC FLAGS_REG))]
-"(optimize_function_for_size_p (cfun)
+"(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-|| !TARGET_PARTIAL_REG_STALL
+/* FIXME: without this LRA can't reload this pattern, see PR82524.  */
-|| (operands[1] == const1_rtx
+&& rtx_equal_p (operands[0], operands[1])"
-	&& TARGET_SHIFT1))"
+{
-{
+if (operands[2] == const1_rtx
-if (operands[1] == const1_rtx
 && (TARGET_SHIFT1 || optimize_function_for_size_p (cfun)))
-return "<rotate>{b}\t%0";
+return "<rotate>{<imodesuffix>}\t%0";
 else
-return "<rotate>{b}\t{%1, %0|%0, %1}";
+return "<rotate>{<imodesuffix>}\t{%2, %0|%0, %2}";
 }
-[(set_attr "type" "rotate1")
+[(set_attr "type" "rotate")
 (set (attr "length_immediate")
 (if_then_else
-(and (match_operand 1 "const1_operand")
+(and (match_operand 2 "const1_operand")
 	    (ior (match_test "TARGET_SHIFT1")
 		 (match_test "optimize_function_for_size_p (cfun)")))
 (const_string "0")
 (const_string "*")))
-(set_attr "mode" "QI")])
+(set_attr "mode" "<MODE>")])
 (define_split
 [(set (match_operand:HI 0 "QIreg_operand")
 (any_rotate:HI (match_dup 0) (const_int 8)))
 (clobber (reg:CC FLAGS_REG))]
 	  (match_operand:SWI48 3 "register_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_BT
 && (INTVAL (operands[2]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_or:SWI48
 	  (match_operand:SWI48 3 "register_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_BT
 && (INTVAL (operands[2]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (any_or:SWI48
 	  (match_operand:SWI48 3 "register_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_BT
 && (INTVAL (operands[2]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (and:SWI48
 	  (match_operand:SWI48 3 "register_operand")))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_BT
 && (INTVAL (operands[2]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(parallel
 [(set (match_dup 0)
 	   (and:SWI48
 && (CONST_INT_P (operands[2])
 ? (INTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)
 	  && INTVAL (operands[2])
 	       >= (optimize_function_for_size_p (cfun) ? 8 : 32))
 : !memory_operand (operands[1], <MODE>mode))
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	  (zero_extract:SWI48
 			 (const_int 0)])
 		      (label_ref (match_operand 3))
 		      (pc)))
 (clobber (reg:CC FLAGS_REG))]
 "(TARGET_USE_BT || optimize_function_for_size_p (cfun))
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	  (zero_extract:SWI48
 		      (pc)))
 (clobber (reg:CC FLAGS_REG))]
 "(TARGET_USE_BT || optimize_function_for_size_p (cfun))
 && (INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode)-1))
 == GET_MODE_BITSIZE (<MODE>mode)-1
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(set (reg:CCC FLAGS_REG)
 	(compare:CCC
 	  (zero_extract:SWI48
 "set%C1\t%0"
 [(set_attr "type" "setcc")
 (set_attr "mode" "QI")])
 (define_insn "*setcc_qi_slp"
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
+[(set (strict_low_part (match_operand:QI 0 "register_operand" "+q"))
 	(match_operator:QI 1 "ix86_comparison_operator"
 	  [(reg FLAGS_REG) (const_int 0)]))]
 ""
 "set%C1\t%0"
 [(set_attr "type" "setcc")
 operands[1] = shallow_copy_rtx (operands[1]);
 PUT_MODE (operands[1], QImode);
 })
 (define_split
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand"))
+[(set (strict_low_part (match_operand:QI 0 "register_operand"))
 	(ne:QI (match_operator 1 "ix86_comparison_operator"
 	         [(reg FLAGS_REG) (const_int 0)])
 	    (const_int 0)))]
 ""
 [(set (match_dup 0) (match_dup 1))]
 if (! ix86_comparison_operator (operands[1], VOIDmode))
 FAIL;
 })
 (define_split
-[(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand"))
+[(set (strict_low_part (match_operand:QI 0 "register_operand"))
 	(eq:QI (match_operator 1 "ix86_comparison_operator"
 	         [(reg FLAGS_REG) (const_int 0)])
 	    (const_int 0)))]
 ""
 [(set (match_dup 0) (match_dup 1))]
 "#"
 "epilogue_completed"
 [(const_int 0)]
 "ix86_expand_epilogue (2); DONE;")
-(define_insn "leave"
+(define_expand "@leave_<mode>"
+[(parallel
+[(set (reg:W SP_REG) (plus:W (reg:W BP_REG) (match_dup 0)))
+(set (reg:W BP_REG) (mem:W (reg:W BP_REG)))
+(clobber (mem:BLK (scratch)))])]
+""
+"operands[0] = GEN_INT (<MODE_SIZE>);")
+(define_insn "*leave"
 [(set (reg:SI SP_REG) (plus:SI (reg:SI BP_REG) (const_int 4)))
 (set (reg:SI BP_REG) (mem:SI (reg:SI BP_REG)))
 (clobber (mem:BLK (scratch)))]
 "!TARGET_64BIT"
 "leave"
 [(set_attr "type" "leave")])
-(define_insn "leave_rex64"
+(define_insn "*leave_rex64"
 [(set (reg:DI SP_REG) (plus:DI (reg:DI BP_REG) (const_int 8)))
 (set (reg:DI BP_REG) (mem:DI (reg:DI BP_REG)))
 (clobber (mem:BLK (scratch)))]
 "TARGET_64BIT"
 "leave"
 ;; BMI2 instructions.
 (define_expand "bmi2_bzhi_<mode>3"
 [(parallel
 [(set (match_operand:SWI48 0 "register_operand")
-	  (zero_extract:SWI48
+	  (if_then_else:SWI48
-	    (match_operand:SWI48 1 "nonimmediate_operand")
+	    (ne:QI (and:SWI48 (match_operand:SWI48 2 "register_operand")
-	    (umin:SWI48
+			      (const_int 255))
-	      (and:SWI48 (match_operand:SWI48 2 "register_operand")
+		   (const_int 0))
-			 (const_int 255))
+	    (zero_extract:SWI48
-	      (match_dup 3))
+	      (match_operand:SWI48 1 "nonimmediate_operand")
+	      (umin:SWI48 (and:SWI48 (match_dup 2) (const_int 255))
+			  (match_dup 3))
+	      (const_int 0))
 	    (const_int 0)))
 (clobber (reg:CC FLAGS_REG))])]
 "TARGET_BMI2"
 "operands[3] = GEN_INT (<MODE_SIZE> * BITS_PER_UNIT);")
 (define_insn "*bmi2_bzhi_<mode>3"
 [(set (match_operand:SWI48 0 "register_operand" "=r")
-	(zero_extract:SWI48
+	(if_then_else:SWI48
-	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")
+	  (ne:QI (and:SWI48 (match_operand:SWI48 2 "register_operand" "r")
-	  (umin:SWI48
+			    (const_int 255))
-	    (and:SWI48 (match_operand:SWI48 2 "register_operand" "r")
+		 (const_int 0))
-		       (const_int 255))
+	  (zero_extract:SWI48
-	    (match_operand:SWI48 3 "const_int_operand" "n"))
+	    (match_operand:SWI48 1 "nonimmediate_operand" "rm")
+	    (umin:SWI48 (and:SWI48 (match_dup 2) (const_int 255))
+			(match_operand:SWI48 3 "const_int_operand" "n"))
+	    (const_int 0))
 	  (const_int 0)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_BMI2 && INTVAL (operands[3]) == <MODE_SIZE> * BITS_PER_UNIT"
 "bzhi\t{%2, %1, %0|%0, %1, %2}"
 [(set_attr "type" "bitmanip")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 (define_insn "*bmi2_bzhi_<mode>3_1"
 [(set (match_operand:SWI48 0 "register_operand" "=r")
-	(zero_extract:SWI48
+	(if_then_else:SWI48
-	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")
+	  (ne:QI (match_operand:QI 2 "register_operand" "r") (const_int 0))
-	  (umin:SWI48
+	  (zero_extract:SWI48
-	    (zero_extend:SWI48 (match_operand:QI 2 "register_operand" "r"))
+	    (match_operand:SWI48 1 "nonimmediate_operand" "rm")
-	    (match_operand:SWI48 3 "const_int_operand" "n"))
+	    (umin:SWI48 (zero_extend:SWI48 (match_dup 2))
+			(match_operand:SWI48 3 "const_int_operand" "n"))
+	    (const_int 0))
 	  (const_int 0)))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_BMI2 && INTVAL (operands[3]) == <MODE_SIZE> * BITS_PER_UNIT"
 "bzhi\t{%<k>2, %1, %0|%0, %1, %<k>2}"
 [(set_attr "type" "bitmanip")
 (set_attr "mode" "<MODE>")])
 (define_insn "*bmi2_bzhi_<mode>3_1_ccz"
 [(set (reg:CCZ FLAGS_REG)
 	(compare:CCZ
-	  (zero_extract:SWI48
+	  (if_then_else:SWI48
-	    (match_operand:SWI48 1 "nonimmediate_operand" "rm")
+	    (ne:QI (match_operand:QI 2 "register_operand" "r") (const_int 0))
-	    (umin:SWI48
+	    (zero_extract:SWI48
-	      (zero_extend:SWI48 (match_operand:QI 2 "register_operand" "r"))
+	      (match_operand:SWI48 1 "nonimmediate_operand" "rm")
-	      (match_operand:SWI48 3 "const_int_operand" "n"))
+	      (umin:SWI48 (zero_extend:SWI48 (match_dup 2))
+			  (match_operand:SWI48 3 "const_int_operand" "n"))
+	      (const_int 0))
 	    (const_int 0))
 	(const_int 0)))
 (clobber (match_scratch:SWI48 0 "=r"))]
 "TARGET_BMI2 && INTVAL (operands[3]) == <MODE_SIZE> * BITS_PER_UNIT"
+"bzhi\t{%<k>2, %1, %0|%0, %1, %<k>2}"
+[(set_attr "type" "bitmanip")
+(set_attr "prefix" "vex")
+(set_attr "mode" "<MODE>")])
+(define_insn "*bmi2_bzhi_<mode>3_2"
+[(set (match_operand:SWI48 0 "register_operand" "=r")
+	(and:SWI48
+	  (plus:SWI48
+	    (ashift:SWI48 (const_int 1)
+			  (match_operand:QI 2 "register_operand" "r"))
+	    (const_int -1))
+	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_BMI2"
+"bzhi\t{%<k>2, %1, %0|%0, %1, %<k>2}"
+[(set_attr "type" "bitmanip")
+(set_attr "prefix" "vex")
+(set_attr "mode" "<MODE>")])
+(define_insn "*bmi2_bzhi_<mode>3_3"
+[(set (match_operand:SWI48 0 "register_operand" "=r")
+	(and:SWI48
+	  (not:SWI48
+	    (ashift:SWI48 (const_int -1)
+			  (match_operand:QI 2 "register_operand" "r")))
+	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_BMI2"
 "bzhi\t{%<k>2, %1, %0|%0, %1, %<k>2}"
 [(set_attr "type" "bitmanip")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 [(set_attr "type" "bitmanip")
 (set_attr "prefix" "vex")
 (set_attr "mode" "<MODE>")])
 ;; TBM instructions.
-(define_insn "tbm_bextri_<mode>"
+(define_expand "tbm_bextri_<mode>"
+[(parallel
+[(set (match_operand:SWI48 0 "register_operand")
+	  (zero_extract:SWI48
+	    (match_operand:SWI48 1 "nonimmediate_operand")
+	    (match_operand 2 "const_0_to_255_operand" "N")
+	    (match_operand 3 "const_0_to_255_operand" "N")))
+(clobber (reg:CC FLAGS_REG))])]
+"TARGET_TBM"
+{
+if (operands[2] == const0_rtx
+|| INTVAL (operands[3]) >= <MODE_SIZE> * BITS_PER_UNIT)
+{
+emit_move_insn (operands[0], const0_rtx);
+DONE;
+}
+if (INTVAL (operands[2]) + INTVAL (operands[3])
+> <MODE_SIZE> * BITS_PER_UNIT)
+operands[2] = GEN_INT (<MODE_SIZE> * BITS_PER_UNIT - INTVAL (operands[3]));
+})
+(define_insn "*tbm_bextri_<mode>"
 [(set (match_operand:SWI48 0 "register_operand" "=r")
 (zero_extract:SWI48
 (match_operand:SWI48 1 "nonimmediate_operand" "rm")
 (match_operand 2 "const_0_to_255_operand" "N")
 (match_operand 3 "const_0_to_255_operand" "N")))
 }
 [(set_attr "prefix_rep" "1")
 (set_attr "type" "bitmanip")
 (set_attr "mode" "<MODE>")])
+(define_insn_and_split "*popcountsi2_zext"
+[(set (match_operand:DI 0 "register_operand" "=r")
+	(and:DI
+	  (subreg:DI
+	    (popcount:SI
+	      (match_operand:SI 1 "nonimmediate_operand" "rm")) 0)
+	  (const_int 63)))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_POPCNT && TARGET_64BIT"
+{
+#if TARGET_MACHO
+return "popcnt\t{%1, %k0|%k0, %1}";
+#else
+return "popcnt{l}\t{%1, %k0|%k0, %1}";
+#endif
+}
+"&& TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+&& optimize_function_for_speed_p (cfun)
+&& !reg_mentioned_p (operands[0], operands[1])"
+[(parallel
+[(set (match_dup 0)
+	  (and:DI (subreg:DI (popcount:SI (match_dup 1)) 0) (const_int 63)))
+(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
+(clobber (reg:CC FLAGS_REG))])]
+"ix86_expand_clear (operands[0]);"
+[(set_attr "prefix_rep" "1")
+(set_attr "type" "bitmanip")
+(set_attr "mode" "SI")])
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt.  There is no false dependency when destination is
+; also used in source.
+(define_insn "*popcountsi2_zext_falsedep"
+[(set (match_operand:DI 0 "register_operand" "=r")
+	(and:DI
+	  (subreg:DI
+	    (popcount:SI
+	      (match_operand:SI 1 "nonimmediate_operand" "rm")) 0)
+	  (const_int 63)))
+(unspec [(match_operand:DI 2 "register_operand" "0")]
+	   UNSPEC_INSN_FALSE_DEP)
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_POPCNT && TARGET_64BIT"
+{
+#if TARGET_MACHO
+return "popcnt\t{%1, %k0|%k0, %1}";
+#else
+return "popcnt{l}\t{%1, %k0|%k0, %1}";
+#endif
+}
+[(set_attr "prefix_rep" "1")
+(set_attr "type" "bitmanip")
+(set_attr "mode" "SI")])
 (define_insn_and_split "*popcounthi2_1"
 [(set (match_operand:SI 0 "register_operand")
 	(popcount:SI
 	  (zero_extend:SI (match_operand:HI 1 "nonimmediate_operand"))))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_POPCNT
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 rtx tmp = gen_reg_rtx (HImode);
 return "call\t{*%%rax|rax}";
 }
 [(set_attr "type" "multi")
 (set_attr "length" "22")])
-(define_expand "tls_global_dynamic_64_<mode>"
+(define_expand "@tls_global_dynamic_64_<mode>"
 [(parallel
 [(set (match_operand:P 0 "register_operand")
 	  (call:P
 	   (mem:QI (match_operand 2))
 	   (const_int 0)))
 return "call\t{*%%rax|rax}";
 }
 [(set_attr "type" "multi")
 (set_attr "length" "22")])
-(define_expand "tls_local_dynamic_base_64_<mode>"
+(define_expand "@tls_local_dynamic_base_64_<mode>"
 [(parallel
 [(set (match_operand:P 0 "register_operand")
 	   (call:P
 	    (mem:QI (match_operand 1))
 	    (const_int 0)))
 {
 operands[5] = can_create_pseudo_p () ? gen_reg_rtx (Pmode) : operands[0];
 emit_insn (gen_tls_dynamic_gnu2_32 (operands[5], operands[1], operands[2]));
 })
-(define_expand "tls_dynamic_gnu2_64"
+(define_expand "@tls_dynamic_gnu2_64_<mode>"
 [(set (match_dup 2)
-	(unspec:DI [(match_operand 1 "tls_symbolic_operand")]
+	(unspec:PTR [(match_operand 1 "tls_symbolic_operand")]
-		   UNSPEC_TLSDESC))
+		    UNSPEC_TLSDESC))
 (parallel
-[(set (match_operand:DI 0 "register_operand")
+[(set (match_operand:PTR 0 "register_operand")
-	  (unspec:DI [(match_dup 1) (match_dup 2) (reg:DI SP_REG)]
+	  (unspec:PTR [(match_dup 1) (match_dup 2) (reg:PTR SP_REG)]
-		     UNSPEC_TLSDESC))
+		      UNSPEC_TLSDESC))
 (clobber (reg:CC FLAGS_REG))])]
 "TARGET_64BIT && TARGET_GNU2_TLS"
 {
-operands[2] = can_create_pseudo_p () ? gen_reg_rtx (Pmode) : operands[0];
+operands[2] = can_create_pseudo_p () ? gen_reg_rtx (ptr_mode) : operands[0];
 ix86_tls_descriptor_calls_expanded_in_cfun = true;
 })
-(define_insn "*tls_dynamic_gnu2_lea_64"
+(define_insn "*tls_dynamic_gnu2_lea_64_<mode>"
-[(set (match_operand:DI 0 "register_operand" "=r")
+[(set (match_operand:PTR 0 "register_operand" "=r")
-	(unspec:DI [(match_operand 1 "tls_symbolic_operand")]
+	(unspec:PTR [(match_operand 1 "tls_symbolic_operand")]
-		   UNSPEC_TLSDESC))]
+		    UNSPEC_TLSDESC))]
 "TARGET_64BIT && TARGET_GNU2_TLS"
-"lea{q}\t{%E1@TLSDESC(%%rip), %0|%0, %E1@TLSDESC[rip]}"
+"lea%z0\t{%E1@TLSDESC(%%rip), %0|%0, %E1@TLSDESC[rip]}"
 [(set_attr "type" "lea")
-(set_attr "mode" "DI")
+(set_attr "mode" "<MODE>")
 (set_attr "length" "7")
 (set_attr "length_address" "4")])
-(define_insn "*tls_dynamic_gnu2_call_64"
+(define_insn "*tls_dynamic_gnu2_call_64_<mode>"
-[(set (match_operand:DI 0 "register_operand" "=a")
+[(set (match_operand:PTR 0 "register_operand" "=a")
-	(unspec:DI [(match_operand 1 "tls_symbolic_operand")
+	(unspec:PTR [(match_operand 1 "tls_symbolic_operand")
-		    (match_operand:DI 2 "register_operand" "0")
+		   (match_operand:PTR 2 "register_operand" "0")
-		    (reg:DI SP_REG)]
+		   (reg:PTR SP_REG)]
-		   UNSPEC_TLSDESC))
+		  UNSPEC_TLSDESC))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT && TARGET_GNU2_TLS"
 "call\t{*%a1@TLSCALL(%2)|[QWORD PTR [%2+%a1@TLSCALL]]}"
 [(set_attr "type" "call")
 (set_attr "length" "2")
 (set_attr "length_address" "0")])
-(define_insn_and_split "*tls_dynamic_gnu2_combine_64"
+(define_insn_and_split "*tls_dynamic_gnu2_combine_64_<mode>"
-[(set (match_operand:DI 0 "register_operand" "=&a")
+[(set (match_operand:PTR 0 "register_operand" "=&a")
-	(plus:DI
+	(plus:PTR
-	 (unspec:DI [(match_operand 2 "tls_modbase_operand")
+	 (unspec:PTR [(match_operand 2 "tls_modbase_operand")
-		     (match_operand:DI 3)
+		      (match_operand:PTR 3)
-		     (reg:DI SP_REG)]
+		      (reg:PTR SP_REG)]
-		    UNSPEC_TLSDESC)
+		     UNSPEC_TLSDESC)
-	 (const:DI (unspec:DI
+	 (const:PTR (unspec:PTR
-		    [(match_operand 1 "tls_symbolic_operand")]
+		     [(match_operand 1 "tls_symbolic_operand")]
-		    UNSPEC_DTPOFF))))
+		     UNSPEC_DTPOFF))))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_64BIT && TARGET_GNU2_TLS"
 "#"
 ""
 [(set (match_dup 0) (match_dup 4))]
 {
-operands[4] = can_create_pseudo_p () ? gen_reg_rtx (Pmode) : operands[0];
+operands[4] = can_create_pseudo_p () ? gen_reg_rtx (ptr_mode) : operands[0];
-emit_insn (gen_tls_dynamic_gnu2_64 (operands[4], operands[1]));
+emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, operands[4], operands[1]));
 })
 (define_split
 [(match_operand 0 "tls_address_pattern")]
 "TARGET_TLS_DIRECT_SEG_REFS"
 	 (eq_attr "alternative" "0")
 	 (symbol_ref "true")
 	 (symbol_ref "false"))))])
 (define_insn "*rcpsf2_sse"
-[(set (match_operand:SF 0 "register_operand" "=x,x")
+[(set (match_operand:SF 0 "register_operand" "=x,x,x")
-	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "x,m")]
+	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "0,x,m")]
 		   UNSPEC_RCP))]
 "TARGET_SSE && TARGET_SSE_MATH"
 "@
+%vrcpss\t{%d1, %0|%0, %d1}
 %vrcpss\t{%d1, %0|%0, %d1}
 %vrcpss\t{%1, %d0|%d0, %1}"
 [(set_attr "type" "sse")
 (set_attr "atom_sse_attr" "rcp")
 (set_attr "btver2_sse_attr" "rcp")
 (set_attr "prefix" "maybe_vex")
-(set_attr "mode" "SF")])
+(set_attr "mode" "SF")
+(set_attr "avx_partial_xmm_update" "false,false,true")
+(set (attr "preferred_for_speed")
+(cond [(match_test "TARGET_AVX")
+	       (symbol_ref "true")
+	     (eq_attr "alternative" "1,2")
+	       (symbol_ref "!TARGET_SSE_PARTIAL_REG_DEPENDENCY")
+	    ]
+	    (symbol_ref "true")))])
 (define_insn "*fop_xf_1_i387"
 [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(match_operator:XF 3 "binary_fp_operator"
 			[(match_operand:XF 1 "register_operand" "0,f")
 	       (match_operand:X87MODEF 3 "div_operator")
 		 (const_string "fdiv")
 	      ]
 	      (const_string "fop")))
 (set_attr "fp_int_src" "true")
-(set_attr "mode" "<MODE>")])
+(set_attr "mode" "<SWI24:MODE>")])
 (define_insn "*fop_xf_4_i387"
 [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(match_operator:XF 3 "binary_fp_operator"
 	   [(float_extend:XF
 (set_attr "athlon_decode" "direct")
 (set_attr "amdfam10_decode" "direct")
 (set_attr "bdver1_decode" "direct")])
 (define_insn "*rsqrtsf2_sse"
-[(set (match_operand:SF 0 "register_operand" "=x,x")
+[(set (match_operand:SF 0 "register_operand" "=x,x,x")
-	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "x,m")]
+	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "0,x,m")]
 		   UNSPEC_RSQRT))]
 "TARGET_SSE && TARGET_SSE_MATH"
 "@
+%vrsqrtss\t{%d1, %0|%0, %d1}
 %vrsqrtss\t{%d1, %0|%0, %d1}
 %vrsqrtss\t{%1, %d0|%d0, %1}"
 [(set_attr "type" "sse")
 (set_attr "atom_sse_attr" "rcp")
 (set_attr "btver2_sse_attr" "rcp")
 (set_attr "prefix" "maybe_vex")
-(set_attr "mode" "SF")])
+(set_attr "mode" "SF")
+(set_attr "avx_partial_xmm_update" "false,false,true")
+(set (attr "preferred_for_speed")
+(cond [(match_test "TARGET_AVX")
+	       (symbol_ref "true")
+	     (eq_attr "alternative" "1,2")
+	       (symbol_ref "!TARGET_SSE_PARTIAL_REG_DEPENDENCY")
+	    ]
+	    (symbol_ref "true")))])
 (define_expand "rsqrtsf2"
 [(set (match_operand:SF 0 "register_operand")
 	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand")]
 		   UNSPEC_RSQRT))]
 ix86_emit_swsqrtsf (operands[0], operands[1], SFmode, 1);
 DONE;
 })
 (define_insn "*sqrt<mode>2_sse"
-[(set (match_operand:MODEF 0 "register_operand" "=v,v")
+[(set (match_operand:MODEF 0 "register_operand" "=v,v,v")
 	(sqrt:MODEF
-	  (match_operand:MODEF 1 "nonimmediate_operand" "v,m")))]
+	  (match_operand:MODEF 1 "nonimmediate_operand" "0,v,m")))]
 "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
 "@
+%vsqrt<ssemodesuffix>\t{%d1, %0|%0, %d1}
 %vsqrt<ssemodesuffix>\t{%d1, %0|%0, %d1}
 %vsqrt<ssemodesuffix>\t{%1, %d0|%d0, %1}"
 [(set_attr "type" "sse")
 (set_attr "atom_sse_attr" "sqrt")
 (set_attr "btver2_sse_attr" "sqrt")
 (set_attr "prefix" "maybe_vex")
+(set_attr "avx_partial_xmm_update" "false,false,true")
 (set_attr "mode" "<MODE>")
-(set_attr "athlon_decode" "*")
+(set (attr "preferred_for_speed")
-(set_attr "amdfam10_decode" "*")
+(cond [(match_test "TARGET_AVX")
-(set_attr "bdver1_decode" "*")])
+	       (symbol_ref "true")
+	     (eq_attr "alternative" "1,2")
+	       (symbol_ref "!TARGET_SSE_PARTIAL_REG_DEPENDENCY")
+	    ]
+	    (symbol_ref "true")))])
 (define_expand "sqrt<mode>2"
 [(set (match_operand:MODEF 0 "register_operand")
 	(sqrt:MODEF
 	  (match_operand:MODEF 1 "nonimmediate_operand")))]
 emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
 emit_insn (gen_sqrtxf2 (op0, op1));
 emit_insn (gen_truncxf<mode>2_i387_noop_unspec (operands[0], op0));
 DONE;
 }
+})
+(define_expand "hypot<mode>3"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))
+(use (match_operand:MODEF 2 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_finite_math_only
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+rtx op2 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op2, operands[2]));
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_mulxf3 (op1, op1, op1));
+emit_insn (gen_mulxf3 (op2, op2, op2));
+emit_insn (gen_addxf3 (op0, op2, op1));
+emit_insn (gen_sqrtxf2 (op0, op0));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
 })
 (define_insn "x86_fnstsw_1"
 [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI [(reg:CCFP FPSR_REG)] UNSPEC_FNSTSW))]
 rtx op0 = gen_reg_rtx (XFmode);
 rtx op1 = gen_reg_rtx (XFmode);
 emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
 emit_insn (gen_acosxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "sinhxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_finite_math_only
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_sinh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "sinh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_finite_math_only
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_sinhxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "coshxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_cosh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "cosh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_coshxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "tanhxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_tanh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "tanh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_tanhxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "asinhxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_finite_math_only
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_asinh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "asinh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_finite_math_only
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_asinhxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "acoshxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_acosh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "acosh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_acoshxf2 (op0, op1));
+emit_insn (gen_truncxf<mode>2 (operands[0], op0));
+DONE;
+})
+(define_expand "atanhxf2"
+[(use (match_operand:XF 0 "register_operand"))
+(use (match_operand:XF 1 "register_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& flag_unsafe_math_optimizations"
+{
+ix86_emit_i387_atanh (operands[0], operands[1]);
+DONE;
+})
+(define_expand "atanh<mode>2"
+[(use (match_operand:MODEF 0 "register_operand"))
+(use (match_operand:MODEF 1 "general_operand"))]
+"TARGET_USE_FANCY_MATH_387
+&& (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+|| TARGET_MIX_SSE_I387)
+&& flag_unsafe_math_optimizations"
+{
+rtx op0 = gen_reg_rtx (XFmode);
+rtx op1 = gen_reg_rtx (XFmode);
+emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
+emit_insn (gen_atanhxf2 (op0, op1));
 emit_insn (gen_truncxf<mode>2 (operands[0], op0));
 DONE;
 })
 (define_insn "fyl2xxf3_i387"
 DONE;
 })
 (define_insn "sse4_1_round<mode>2"
-[(set (match_operand:MODEF 0 "register_operand" "=x,v")
+[(set (match_operand:MODEF 0 "register_operand" "=x,x,x,v,v")
-	(unspec:MODEF [(match_operand:MODEF 1 "nonimmediate_operand" "xm,vm")
+	(unspec:MODEF
-		       (match_operand:SI 2 "const_0_to_15_operand" "n,n")]
+	  [(match_operand:MODEF 1 "nonimmediate_operand" "0,x,m,v,m")
-		      UNSPEC_ROUND))]
+	   (match_operand:SI 2 "const_0_to_15_operand" "n,n,n,n,n")]
+	  UNSPEC_ROUND))]
 "TARGET_SSE4_1"
 "@
+%vround<ssemodesuffix>\t{%2, %d1, %0|%0, %d1, %2}
+%vround<ssemodesuffix>\t{%2, %d1, %0|%0, %d1, %2}
 %vround<ssemodesuffix>\t{%2, %1, %d0|%d0, %1, %2}
+vrndscale<ssemodesuffix>\t{%2, %d1, %0|%0, %d1, %2}
 vrndscale<ssemodesuffix>\t{%2, %1, %d0|%d0, %1, %2}"
 [(set_attr "type" "ssecvt")
-(set_attr "prefix_extra" "1,*")
+(set_attr "prefix_extra" "1,1,1,*,*")
-(set_attr "length_immediate" "*,1")
+(set_attr "length_immediate" "*,*,*,1,1")
-(set_attr "prefix" "maybe_vex,evex")
+(set_attr "prefix" "maybe_vex,maybe_vex,maybe_vex,evex,evex")
-(set_attr "isa" "noavx512f,avx512f")
+(set_attr "isa" "noavx512f,noavx512f,noavx512f,avx512f,avx512f")
-(set_attr "mode" "<MODE>")])
+(set_attr "avx_partial_xmm_update" "false,false,true,false,true")
+(set_attr "mode" "<MODE>")
+(set (attr "preferred_for_speed")
+(cond [(match_test "TARGET_AVX")
+	       (symbol_ref "true")
+	     (eq_attr "alternative" "1,2")
+	       (symbol_ref "!TARGET_SSE_PARTIAL_REG_DEPENDENCY")
+	    ]
+	    (symbol_ref "true")))])
 (define_insn "rintxf2"
 [(set (match_operand:XF 0 "register_operand" "=f")
 	(unspec:XF [(match_operand:XF 1 "register_operand" "0")]
 		   UNSPEC_FRNDINT))]
 ix86_emit_i387_round (operands[0], operands[1]);
 DONE;
 })
 (define_int_iterator FRNDINT_ROUNDING
-	[UNSPEC_FRNDINT_FLOOR
+	[UNSPEC_FRNDINT_ROUNDEVEN
+	 UNSPEC_FRNDINT_FLOOR
 	 UNSPEC_FRNDINT_CEIL
 	 UNSPEC_FRNDINT_TRUNC])
 (define_int_iterator FIST_ROUNDING
 	[UNSPEC_FIST_FLOOR
 	 UNSPEC_FIST_CEIL])
 ;; Base name for define_insn
 (define_int_attr rounding_insn
-	[(UNSPEC_FRNDINT_FLOOR "floor")
+	[(UNSPEC_FRNDINT_ROUNDEVEN "roundeven")
+	 (UNSPEC_FRNDINT_FLOOR "floor")
 	 (UNSPEC_FRNDINT_CEIL "ceil")
 	 (UNSPEC_FRNDINT_TRUNC "btrunc")
 	 (UNSPEC_FIST_FLOOR "floor")
 	 (UNSPEC_FIST_CEIL "ceil")])
 (define_int_attr rounding
-	[(UNSPEC_FRNDINT_FLOOR "floor")
+	[(UNSPEC_FRNDINT_ROUNDEVEN "roundeven")
+	 (UNSPEC_FRNDINT_FLOOR "floor")
 	 (UNSPEC_FRNDINT_CEIL "ceil")
 	 (UNSPEC_FRNDINT_TRUNC "trunc")
 	 (UNSPEC_FIST_FLOOR "floor")
 	 (UNSPEC_FIST_CEIL "ceil")])
 (define_int_attr ROUNDING
-	[(UNSPEC_FRNDINT_FLOOR "FLOOR")
+	[(UNSPEC_FRNDINT_ROUNDEVEN "ROUNDEVEN")
+	 (UNSPEC_FRNDINT_FLOOR "FLOOR")
 	 (UNSPEC_FRNDINT_CEIL "CEIL")
 	 (UNSPEC_FRNDINT_TRUNC "TRUNC")
 	 (UNSPEC_FIST_FLOOR "FLOOR")
 	 (UNSPEC_FIST_CEIL "CEIL")])
 	(unspec:XF [(match_operand:XF 1 "register_operand")]
 		   FRNDINT_ROUNDING))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_FANCY_MATH_387
 && (flag_fp_int_builtin_inexact || !flag_trapping_math)
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 ix86_optimize_mode_switching[I387_<ROUNDING>] = 1;
 "(TARGET_USE_FANCY_MATH_387
 && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 	|| TARGET_MIX_SSE_I387)
 && (flag_fp_int_builtin_inexact || !flag_trapping_math))
 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
-&& (TARGET_SSE4_1 || flag_fp_int_builtin_inexact
+&& (TARGET_SSE4_1
-	   || !flag_trapping_math))"
+	  || (ROUND_<ROUNDING> != ROUND_ROUNDEVEN
+	      && (flag_fp_int_builtin_inexact || !flag_trapping_math))))"
 {
 if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
 && (TARGET_SSE4_1 || flag_fp_int_builtin_inexact || !flag_trapping_math))
 {
 if (TARGET_SSE4_1)
 	(unspec:SWI248x [(match_operand:XF 1 "register_operand")]
 			FIST_ROUNDING))
 (clobber (reg:CC FLAGS_REG))]
 "TARGET_USE_FANCY_MATH_387
 && flag_unsafe_math_optimizations
-&& can_create_pseudo_p ()"
+&& ix86_pre_reload_split ()"
 "#"
 "&& 1"
 [(const_int 0)]
 {
 ix86_optimize_mode_switching[I387_<ROUNDING>] = 1;
 {
 if (TARGET_SSE4_1)
 {
 rtx tmp = gen_reg_rtx (<MODEF:MODE>mode);
-emit_insn (gen_sse4_1_round<mode>2
+emit_insn (gen_sse4_1_round<MODEF:mode>2
 		 (tmp, operands[1], GEN_INT (ROUND_<ROUNDING>
 					     | ROUND_NO_EXC)));
 emit_insn (gen_fix_trunc<MODEF:mode><SWI48:mode>2
 		 (operands[0], tmp));
 }
 "cld"
 [(set_attr "length" "1")
 (set_attr "length_immediate" "0")
 (set_attr "modrm" "0")])
-(define_expand "movmem<mode>"
+(define_expand "cpymem<mode>"
 [(use (match_operand:BLK 0 "memory_operand"))
 (use (match_operand:BLK 1 "memory_operand"))
 (use (match_operand:SWI48 2 "nonmemory_operand"))
 (use (match_operand:SWI48 3 "const_int_operand"))
 (use (match_operand:SI 4 "const_int_operand"))
 (use (match_operand:SI 6 ""))
 (use (match_operand:SI 7 ""))
 (use (match_operand:SI 8 ""))]
 ""
 {
-if (ix86_expand_set_or_movmem (operands[0], operands[1],
+if (ix86_expand_set_or_cpymem (operands[0], operands[1],
 			        operands[2], NULL, operands[3],
 			        operands[4], operands[5],
 				operands[6], operands[7],
 				operands[8], false))
 DONE;
 (use (match_operand:SI 6 ""))
 (use (match_operand:SI 7 ""))
 (use (match_operand:SI 8 ""))]
 ""
 {
-if (ix86_expand_set_or_movmem (operands[0], NULL,
+if (ix86_expand_set_or_cpymem (operands[0], NULL,
 			        operands[1], operands[2],
 				operands[3], operands[4],
 			        operands[5], operands[6],
 				operands[7], operands[8], true))
 DONE;
 		    (match_operand:BLK 2 "general_operand")))
 (use (match_operand 3 "general_operand"))
 (use (match_operand 4 "immediate_operand"))]
 ""
 {
-rtx addr1, addr2, out, outlow, count, countreg, align;
+rtx addr1, addr2, countreg, align, out;
 if (optimize_insn_for_size_p () && !TARGET_INLINE_ALL_STRINGOPS)
 FAIL;
 /* Can't use this if the user has appropriated ecx, esi or edi.  */
 || (t2 && TREE_CODE (t2) == MEM_REF
 && TREE_CODE (TREE_OPERAND (t2, 0)) == ADDR_EXPR
 && TREE_CODE (TREE_OPERAND (TREE_OPERAND (t2, 0), 0)) == STRING_CST)))
 FAIL;
-out = operands[0];
-if (!REG_P (out))
-out = gen_reg_rtx (SImode);
 addr1 = copy_addr_to_reg (XEXP (operands[1], 0));
 addr2 = copy_addr_to_reg (XEXP (operands[2], 0));
 if (addr1 != XEXP (operands[1], 0))
 operands[1] = replace_equiv_address_nv (operands[1], addr1);
 if (addr2 != XEXP (operands[2], 0))
 operands[2] = replace_equiv_address_nv (operands[2], addr2);
-count = operands[3];
+countreg = ix86_zero_extend_to_Pmode (operands[3]);
-countreg = ix86_zero_extend_to_Pmode (count);
 /* %%% Iff we are testing strict equality, we can use known alignment
 to good advantage.  This may be possible with combine, particularly
 once cc0 is dead.  */
 align = operands[4];
-if (CONST_INT_P (count))
+if (CONST_INT_P (operands[3]))
 {
-if (INTVAL (count) == 0)
+if (operands[3] == const0_rtx)
 	{
 	  emit_move_insn (operands[0], const0_rtx);
 	  DONE;
 	}
 emit_insn (gen_cmpstrnqi_nz_1 (addr1, addr2, countreg, align,
 				     operands[1], operands[2]));
 }
 else
 {
-rtx (*gen_cmp) (rtx, rtx);
+emit_insn (gen_cmp_1 (Pmode, countreg, countreg));
-gen_cmp = (TARGET_64BIT
-		 ? gen_cmpdi_1 : gen_cmpsi_1);
-emit_insn (gen_cmp (countreg, countreg));
 emit_insn (gen_cmpstrnqi_1 (addr1, addr2, countreg, align,
 				  operands[1], operands[2]));
 }
-outlow = gen_lowpart (QImode, out);
+out = gen_lowpart (QImode, operands[0]);
-emit_insn (gen_cmpintqi (outlow));
+emit_insn (gen_cmpintqi (out));
-emit_move_insn (out, gen_rtx_SIGN_EXTEND (SImode, outlow));
+emit_move_insn (operands[0], gen_rtx_SIGN_EXTEND (SImode, out));
-if (operands[0] != out)
-emit_move_insn (operands[0], out);
 DONE;
 })
 ;; Produce a tri-state integer (-1, 0, 1) from condition codes.
 (set_attr "use_carry" "1")
 (set_attr "pent_pair" "pu")
 (set_attr "mode" "<MODE>")
 (set_attr "length_immediate" "0")])
+(define_insn_and_split "*x86_mov<SWI48:mode>cc_0_m1_neg_leu<SWI:mode>"
+[(set (match_operand:SWI48 0 "register_operand" "=r")
+	(neg:SWI48
+	  (leu:SWI48
+	    (match_operand:SWI 1 "nonimmediate_operand" "<SWI:r>m")
+	    (match_operand:SWI 2 "<SWI:immediate_operand>" "<SWI:i>"))))
+(clobber (reg:CC FLAGS_REG))]
+"CONST_INT_P (operands[2])
+&& INTVAL (operands[2]) != -1
+&& INTVAL (operands[2]) != 2147483647"
+"#"
+""
+[(set (reg:CC FLAGS_REG) (compare:CC (match_dup 1) (match_dup 2)))
+(parallel [(set (match_dup 0)
+		   (neg:SWI48 (ltu:SWI48 (reg:CC FLAGS_REG) (const_int 0))))
+	      (clobber (reg:CC FLAGS_REG))])]
+"operands[2] = GEN_INT (INTVAL (operands[2]) + 1);")
 (define_insn "*mov<mode>cc_noc"
 [(set (match_operand:SWI248 0 "register_operand" "=r,r")
 	(if_then_else:SWI248 (match_operator 1 "ix86_comparison_operator"
 			       [(reg FLAGS_REG) (const_int 0)])
 	  (match_operand:SWI248 2 "nonimmediate_operand" "rm,0")
 (match_operand 1 "ordered_comparison_operator")
 (match_operand:SWI 2 "register_operand")
 (match_operand:SWI 3 "const_int_operand")]
 ""
 "if (ix86_expand_int_addcc (operands)) DONE; else FAIL;")
+;; min/max patterns
+(define_mode_iterator MAXMIN_IMODE
+[(SI "TARGET_SSE4_1") (DI "TARGET_AVX512VL")])
+(define_code_attr maxmin_rel
+[(smax "GE") (smin "LE") (umax "GEU") (umin "LEU")])
+(define_expand "<code><mode>3"
+[(parallel
+[(set (match_operand:MAXMIN_IMODE 0 "register_operand")
+	  (maxmin:MAXMIN_IMODE
+	    (match_operand:MAXMIN_IMODE 1 "register_operand")
+	    (match_operand:MAXMIN_IMODE 2 "nonimmediate_operand")))
+(clobber (reg:CC FLAGS_REG))])]
+"TARGET_STV")
+(define_insn_and_split "*<code><mode>3_1"
+[(set (match_operand:MAXMIN_IMODE 0 "register_operand")
+	(maxmin:MAXMIN_IMODE
+	  (match_operand:MAXMIN_IMODE 1 "register_operand")
+	  (match_operand:MAXMIN_IMODE 2 "nonimmediate_operand")))
+(clobber (reg:CC FLAGS_REG))]
+"(TARGET_64BIT || <MODE>mode != DImode) && TARGET_STV
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (match_dup 0)
+	(if_then_else:MAXMIN_IMODE (match_dup 3)
+	  (match_dup 1)
+	  (match_dup 2)))]
+{
+machine_mode mode = <MODE>mode;
+if (!register_operand (operands[2], mode))
+operands[2] = force_reg (mode, operands[2]);
+enum rtx_code code = <maxmin_rel>;
+machine_mode cmpmode = SELECT_CC_MODE (code, operands[1], operands[2]);
+rtx flags = gen_rtx_REG (cmpmode, FLAGS_REG);
+rtx tmp = gen_rtx_COMPARE (cmpmode, operands[1], operands[2]);
+emit_insn (gen_rtx_SET (flags, tmp));
+operands[3] = gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
+})
+(define_insn_and_split "*<code>di3_doubleword"
+[(set (match_operand:DI 0 "register_operand")
+	(maxmin:DI (match_operand:DI 1 "register_operand")
+		   (match_operand:DI 2 "nonimmediate_operand")))
+(clobber (reg:CC FLAGS_REG))]
+"!TARGET_64BIT && TARGET_STV && TARGET_AVX512VL
+&& ix86_pre_reload_split ()"
+"#"
+"&& 1"
+[(set (match_dup 0)
+	(if_then_else:SI (match_dup 6)
+	  (match_dup 1)
+	  (match_dup 2)))
+(set (match_dup 3)
+	(if_then_else:SI (match_dup 6)
+	  (match_dup 4)
+	  (match_dup 5)))]
+{
+if (!register_operand (operands[2], DImode))
+operands[2] = force_reg (DImode, operands[2]);
+split_double_mode (DImode, &operands[0], 3, &operands[0], &operands[3]);
+rtx cmplo[2] = { operands[1], operands[2] };
+rtx cmphi[2] = { operands[4], operands[5] };
+enum rtx_code code = <maxmin_rel>;
+switch (code)
+{
+case LE: case LEU:
+std::swap (cmplo[0], cmplo[1]);
+std::swap (cmphi[0], cmphi[1]);
+code = swap_condition (code);
+/* FALLTHRU */
+case GE: case GEU:
+{
+	bool uns = (code == GEU);
+	rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
+	  = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
+	emit_insn (gen_cmp_1 (SImode, cmplo[0], cmplo[1]));
+	rtx tmp = gen_rtx_SCRATCH (SImode);
+	emit_insn (sbb_insn (SImode, tmp, cmphi[0], cmphi[1]));
+	rtx flags = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
+	operands[6] = gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
+	break;
+}
+default:
+gcc_unreachable ();
+}
+})
 ;; Misc patterns (?)
 ;; This pattern exists to put a dependency on all ebp-based memory accesses.
 ;; Otherwise there will be nothing to keep
 ;;  (clobber (eflags)]
 ;; [(set (mem (plus (reg ebp) (const_int -160000))) (const_int 0))]
 ;;
 ;; in proper program order.
-(define_insn "pro_epilogue_adjust_stack_<mode>_add"
+(define_insn "@pro_epilogue_adjust_stack_add_<mode>"
 [(set (match_operand:P 0 "register_operand" "=r,r")
 	(plus:P (match_operand:P 1 "register_operand" "0,r")
 	        (match_operand:P 2 "<nonmemory_operand>" "r<i>,l<i>")))
 (clobber (reg:CC FLAGS_REG))
 (clobber (mem:BLK (scratch)))]
 		 (const_string "1")
 	      ]
 	      (const_string "*")))
 (set_attr "mode" "<MODE>")])
-(define_insn "pro_epilogue_adjust_stack_<mode>_sub"
+(define_insn "@pro_epilogue_adjust_stack_sub_<mode>"
 [(set (match_operand:P 0 "register_operand" "=r")
 	(minus:P (match_operand:P 1 "register_operand" "0")
 		 (match_operand:P 2 "register_operand" "r")))
 (clobber (reg:CC FLAGS_REG))
 (clobber (mem:BLK (scratch)))]
 ""
 "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
 [(set_attr "type" "alu")
 (set_attr "mode" "<MODE>")])
-(define_insn "allocate_stack_worker_probe_<mode>"
+(define_insn "@allocate_stack_worker_probe_<mode>"
 [(set (match_operand:P 0 "register_operand" "=a")
 	(unspec_volatile:P [(match_operand:P 1 "register_operand" "0")]
 			    UNSPECV_STACK_PROBE))
 (clobber (reg:CC FLAGS_REG))]
 "ix86_target_stack_probe ()"
 if (CHECK_STACK_LIMIT && CONST_INT_P (operands[1])
 && INTVAL (operands[1]) < CHECK_STACK_LIMIT)
 x = operands[1];
 else
 {
-rtx (*insn) (rtx, rtx);
 x = copy_to_mode_reg (Pmode, operands[1]);
-insn = (TARGET_64BIT
+emit_insn (gen_allocate_stack_worker_probe (Pmode, x, x));
-	      ? gen_allocate_stack_worker_probe_di
-	      : gen_allocate_stack_worker_probe_si);
-emit_insn (insn (x, x));
 }
 x = expand_simple_binop (Pmode, MINUS, stack_pointer_rtx, x,
 			   stack_pointer_rtx, 0, OPTAB_DIRECT);
 (define_expand "probe_stack"
 [(match_operand 0 "memory_operand")]
 ""
 {
-rtx (*insn) (rtx, rtx)
+emit_insn (gen_probe_stack_1
-= (GET_MODE (operands[0]) == DImode
+	     (word_mode, operands[0], const0_rtx));
-? gen_probe_stack_di : gen_probe_stack_si);
-emit_insn (insn (operands[0], const0_rtx));
 DONE;
 })
 ;; Use OR for stack probes, this is shorter.
-(define_insn "probe_stack_<mode>"
+(define_insn "@probe_stack_1_<mode>"
 [(set (match_operand:W 0 "memory_operand" "=m")
 	(unspec:W [(match_operand:W 1 "const0_operand")]
 		  UNSPEC_PROBE_STACK))
 (clobber (reg:CC FLAGS_REG))]
 ""
 "or{<imodesuffix>}\t{%1, %0|%0, %1}"
 [(set_attr "type" "alu1")
 (set_attr "mode" "<MODE>")
 (set_attr "length_immediate" "1")])
-(define_insn "adjust_stack_and_probe<mode>"
+(define_insn "@adjust_stack_and_probe_<mode>"
 [(set (match_operand:P 0 "register_operand" "=r")
 	(unspec_volatile:P [(match_operand:P 1 "register_operand" "0")]
 			    UNSPECV_PROBE_STACK_RANGE))
 (set (reg:P SP_REG)
 (minus:P (reg:P SP_REG) (match_operand:P 2 "const_int_operand" "n")))
 (clobber (mem:BLK (scratch)))]
 ""
 "* return output_adjust_stack_and_probe (operands[0]);"
 [(set_attr "type" "multi")])
-(define_insn "probe_stack_range<mode>"
+(define_insn "@probe_stack_range_<mode>"
 [(set (match_operand:P 0 "register_operand" "=r")
 	(unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
 			    (match_operand:P 2 "const_int_operand" "n")]
 			    UNSPECV_PROBE_STACK_RANGE))
 (clobber (reg:CC FLAGS_REG))]
 ;; Likewise for instances where we have a lea pattern.
 (define_peephole2
 [(set (match_operand:SWI 0 "register_operand")
 	(match_operand:SWI 1 "memory_operand"))
-(set (match_operand:SWI 3 "register_operand")
+(set (match_operand:<LEAMODE> 3 "register_operand")
-	(plus:SWI (match_dup 0)
+	(plus:<LEAMODE> (match_operand:<LEAMODE> 4 "register_operand")
-		  (match_operand:SWI 2 "<nonmemory_operand>")))
+			(match_operand:<LEAMODE> 2 "<nonmemory_operand>")))
-(set (match_dup 1) (match_dup 3))
+(set (match_dup 1) (match_operand:SWI 5 "register_operand"))
-(set (reg FLAGS_REG) (compare (match_dup 3) (const_int 0)))]
+(set (reg FLAGS_REG) (compare (match_dup 5) (const_int 0)))]
 "(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& REGNO (operands[4]) == REGNO (operands[0])
+&& REGNO (operands[5]) == REGNO (operands[3])
 && peep2_reg_dead_p (4, operands[3])
-&& (rtx_equal_p (operands[0], operands[3])
+&& ((REGNO (operands[0]) == REGNO (operands[3]))
 || peep2_reg_dead_p (2, operands[0]))
 && !reg_overlap_mentioned_p (operands[0], operands[1])
 && !reg_overlap_mentioned_p (operands[3], operands[1])
 && !reg_overlap_mentioned_p (operands[0], operands[2])
 && (<MODE>mode != QImode
 || immediate_operand (operands[2], QImode)
 || any_QIreg_operand (operands[2], QImode))
 && ix86_match_ccmode (peep2_next_insn (3), CCGOCmode)"
-[(parallel [(set (match_dup 4) (match_dup 6))
+[(parallel [(set (match_dup 6) (match_dup 8))
-	      (set (match_dup 1) (match_dup 5))])]
+	      (set (match_dup 1) (match_dup 7))])]
 {
-operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
+operands[6] = SET_DEST (PATTERN (peep2_next_insn (3)));
-operands[5]
+operands[7]
 = gen_rtx_PLUS (<MODE>mode,
 		    copy_rtx (operands[1]),
-		    operands[2]);
+		    gen_lowpart (<MODE>mode, operands[2]));
-operands[6]
+operands[8]
-= gen_rtx_COMPARE (GET_MODE (operands[4]),
+= gen_rtx_COMPARE (GET_MODE (operands[6]),
-		       copy_rtx (operands[5]),
+		       copy_rtx (operands[7]),
 		       const0_rtx);
 })
 (define_peephole2
 [(parallel [(set (match_operand:SWI 0 "register_operand")
 && !reg_overlap_mentioned_p (operands[0], operands[2])
 && ix86_match_ccmode (peep2_next_insn (3),
 			 (GET_CODE (operands[3]) == PLUS
 			  || GET_CODE (operands[3]) == MINUS)
 			 ? CCGOCmode : CCNOmode)"
-[(parallel [(set (match_dup 4) (match_dup 6))
+[(parallel [(set (match_dup 5) (match_dup 7))
-	      (set (match_dup 1) (match_dup 5))])]
+	      (set (match_dup 1) (match_dup 6))])]
 {
-operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
+operands[5] = SET_DEST (PATTERN (peep2_next_insn (3)));
-operands[5]
+operands[6]
 = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
 		      copy_rtx (operands[1]),
 		      gen_lowpart (<MODE>mode, operands[2]));
+operands[7]
+= gen_rtx_COMPARE (GET_MODE (operands[5]),
+		       copy_rtx (operands[6]),
+		       const0_rtx);
+})
+;; peephole2 comes before regcprop, so deal also with a case that
+;; would be cleaned up by regcprop.
+(define_peephole2
+[(set (match_operand:SWI 0 "register_operand")
+	(match_operand:SWI 1 "memory_operand"))
+(parallel [(set (match_dup 0)
+		   (match_operator:SWI 3 "plusminuslogic_operator"
+		     [(match_dup 0)
+		      (match_operand:SWI 2 "<nonmemory_operand>")]))
+	      (clobber (reg:CC FLAGS_REG))])
+(set (match_operand:SWI 4 "register_operand") (match_dup 0))
+(set (match_dup 1) (match_dup 4))
+(set (reg FLAGS_REG) (compare (match_dup 4) (const_int 0)))]
+"(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& peep2_reg_dead_p (3, operands[0])
+&& peep2_reg_dead_p (5, operands[4])
+&& !reg_overlap_mentioned_p (operands[0], operands[1])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])
+&& !reg_overlap_mentioned_p (operands[4], operands[1])
+&& (<MODE>mode != QImode
+|| immediate_operand (operands[2], QImode)
+|| any_QIreg_operand (operands[2], QImode))
+&& ix86_match_ccmode (peep2_next_insn (4),
+			 (GET_CODE (operands[3]) == PLUS
+			  || GET_CODE (operands[3]) == MINUS)
+			 ? CCGOCmode : CCNOmode)"
+[(parallel [(set (match_dup 5) (match_dup 7))
+	      (set (match_dup 1) (match_dup 6))])]
+{
+operands[5] = SET_DEST (PATTERN (peep2_next_insn (4)));
 operands[6]
-= gen_rtx_COMPARE (GET_MODE (operands[4]),
+= gen_rtx_fmt_ee (GET_CODE (operands[3]), GET_MODE (operands[3]),
-		       copy_rtx (operands[5]),
+		      copy_rtx (operands[1]),
+		      operands[2]);
+operands[7]
+= gen_rtx_COMPARE (GET_MODE (operands[5]),
+		       copy_rtx (operands[6]),
 		       const0_rtx);
 })
+(define_peephole2
+[(set (match_operand:SWI12 0 "register_operand")
+	(match_operand:SWI12 1 "memory_operand"))
+(parallel [(set (match_operand:SI 4 "register_operand")
+		   (match_operator:SI 3 "plusminuslogic_operator"
+		     [(match_dup 4)
+		      (match_operand:SI 2 "nonmemory_operand")]))
+	      (clobber (reg:CC FLAGS_REG))])
+(set (match_operand:SWI12 5 "register_operand") (match_dup 0))
+(set (match_dup 1) (match_dup 5))
+(set (reg FLAGS_REG) (compare (match_dup 5) (const_int 0)))]
+"(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& REGNO (operands[0]) == REGNO (operands[4])
+&& peep2_reg_dead_p (3, operands[0])
+&& peep2_reg_dead_p (5, operands[5])
+&& (<MODE>mode != QImode
+|| immediate_operand (operands[2], SImode)
+|| any_QIreg_operand (operands[2], SImode))
+&& !reg_overlap_mentioned_p (operands[0], operands[1])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])
+&& !reg_overlap_mentioned_p (operands[5], operands[1])
+&& ix86_match_ccmode (peep2_next_insn (4),
+			 (GET_CODE (operands[3]) == PLUS
+			  || GET_CODE (operands[3]) == MINUS)
+			 ? CCGOCmode : CCNOmode)"
+[(parallel [(set (match_dup 6) (match_dup 8))
+	      (set (match_dup 1) (match_dup 7))])]
+{
+operands[6] = SET_DEST (PATTERN (peep2_next_insn (4)));
+operands[7]
+= gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
+		      copy_rtx (operands[1]),
+		      gen_lowpart (<MODE>mode, operands[2]));
+operands[8]
+= gen_rtx_COMPARE (GET_MODE (operands[6]),
+		       copy_rtx (operands[7]),
+		       const0_rtx);
+})
+;; Likewise for cmpelim optimized pattern.
+(define_peephole2
+[(set (match_operand:SWI 0 "register_operand")
+	(match_operand:SWI 1 "memory_operand"))
+(parallel [(set (reg FLAGS_REG)
+		   (compare (match_operator:SWI 3 "plusminuslogic_operator"
+			      [(match_dup 0)
+			       (match_operand:SWI 2 "<nonmemory_operand>")])
+			    (const_int 0)))
+	      (set (match_dup 0) (match_dup 3))])
+(set (match_operand:SWI 4 "register_operand") (match_dup 0))
+(set (match_dup 1) (match_dup 4))]
+"(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& peep2_reg_dead_p (3, operands[0])
+&& peep2_reg_dead_p (4, operands[4])
+&& !reg_overlap_mentioned_p (operands[0], operands[1])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])
+&& !reg_overlap_mentioned_p (operands[4], operands[1])
+&& ix86_match_ccmode (peep2_next_insn (1),
+			 (GET_CODE (operands[3]) == PLUS
+			  || GET_CODE (operands[3]) == MINUS)
+			 ? CCGOCmode : CCNOmode)"
+[(parallel [(set (match_dup 5) (match_dup 7))
+	      (set (match_dup 1) (match_dup 6))])]
+{
+operands[5] = SET_DEST (XVECEXP (PATTERN (peep2_next_insn (1)), 0, 0));
+operands[6]
+= gen_rtx_fmt_ee (GET_CODE (operands[3]), GET_MODE (operands[3]),
+		      copy_rtx (operands[1]), operands[2]);
+operands[7]
+= gen_rtx_COMPARE (GET_MODE (operands[5]), copy_rtx (operands[6]),
+		       const0_rtx);
+})
+;; Special cases for xor, where (x ^= y) != 0 is (misoptimized)
+;; into x = z; x ^= y; x != z
+(define_peephole2
+[(set (match_operand:SWI 0 "register_operand")
+	(match_operand:SWI 1 "memory_operand"))
+(set (match_operand:SWI 3 "register_operand") (match_dup 0))
+(parallel [(set (match_operand:SWI 4 "register_operand")
+		   (xor:SWI (match_dup 4)
+			    (match_operand:SWI 2 "<nonmemory_operand>")))
+	      (clobber (reg:CC FLAGS_REG))])
+(set (match_dup 1) (match_dup 4))
+(set (reg:CCZ FLAGS_REG)
+	(compare:CCZ (match_operand:SWI 5 "register_operand")
+		     (match_operand:SWI 6 "<nonmemory_operand>")))]
+"(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& (REGNO (operands[4]) == REGNO (operands[0])
+|| REGNO (operands[4]) == REGNO (operands[3]))
+&& (rtx_equal_p (operands[REGNO (operands[4]) == REGNO (operands[0])
+			     ? 3 : 0], operands[5])
+? rtx_equal_p (operands[2], operands[6])
+: rtx_equal_p (operands[2], operands[5])
+	 && rtx_equal_p (operands[REGNO (operands[4]) == REGNO (operands[0])
+				  ? 3 : 0], operands[6]))
+&& peep2_reg_dead_p (4, operands[4])
+&& peep2_reg_dead_p (5, operands[REGNO (operands[4]) == REGNO (operands[0])
+				    ? 3 : 0])
+&& !reg_overlap_mentioned_p (operands[0], operands[1])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])
+&& !reg_overlap_mentioned_p (operands[3], operands[0])
+&& !reg_overlap_mentioned_p (operands[3], operands[1])
+&& !reg_overlap_mentioned_p (operands[3], operands[2])
+&& (<MODE>mode != QImode
+|| immediate_operand (operands[2], QImode)
+|| any_QIreg_operand (operands[2], QImode))"
+[(parallel [(set (match_dup 7) (match_dup 9))
+	      (set (match_dup 1) (match_dup 8))])]
+{
+operands[7] = SET_DEST (PATTERN (peep2_next_insn (4)));
+operands[8] = gen_rtx_XOR (<MODE>mode, copy_rtx (operands[1]),
+			     operands[2]);
+operands[9]
+= gen_rtx_COMPARE (GET_MODE (operands[7]),
+		       copy_rtx (operands[8]),
+		       const0_rtx);
+})
+(define_peephole2
+[(set (match_operand:SWI12 0 "register_operand")
+	(match_operand:SWI12 1 "memory_operand"))
+(set (match_operand:SWI12 3 "register_operand") (match_dup 0))
+(parallel [(set (match_operand:SI 4 "register_operand")
+		   (xor:SI (match_dup 4)
+			   (match_operand:SI 2 "<nonmemory_operand>")))
+	      (clobber (reg:CC FLAGS_REG))])
+(set (match_dup 1) (match_operand:SWI12 5 "register_operand"))
+(set (reg:CCZ FLAGS_REG)
+	(compare:CCZ (match_operand:SWI12 6 "register_operand")
+		     (match_operand:SWI12 7 "<nonmemory_operand>")))]
+"(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
+&& (REGNO (operands[5]) == REGNO (operands[0])
+|| REGNO (operands[5]) == REGNO (operands[3]))
+&& REGNO (operands[5]) == REGNO (operands[4])
+&& (rtx_equal_p (operands[REGNO (operands[5]) == REGNO (operands[0])
+			     ? 3 : 0], operands[6])
+? (REG_P (operands[2])
+	  ? REG_P (operands[7]) && REGNO (operands[2]) == REGNO (operands[7])
+	  : rtx_equal_p (operands[2], operands[7]))
+: (rtx_equal_p (operands[REGNO (operands[5]) == REGNO (operands[0])
+				? 3 : 0], operands[7])
+	  && REG_P (operands[2])
+	  && REGNO (operands[2]) == REGNO (operands[6])))
+&& peep2_reg_dead_p (4, operands[5])
+&& peep2_reg_dead_p (5, operands[REGNO (operands[5]) == REGNO (operands[0])
+				    ? 3 : 0])
+&& !reg_overlap_mentioned_p (operands[0], operands[1])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])
+&& !reg_overlap_mentioned_p (operands[3], operands[0])
+&& !reg_overlap_mentioned_p (operands[3], operands[1])
+&& !reg_overlap_mentioned_p (operands[3], operands[2])
+&& (<MODE>mode != QImode
+|| immediate_operand (operands[2], SImode)
+|| any_QIreg_operand (operands[2], SImode))"
+[(parallel [(set (match_dup 8) (match_dup 10))
+	      (set (match_dup 1) (match_dup 9))])]
+{
+operands[8] = SET_DEST (PATTERN (peep2_next_insn (4)));
+operands[9] = gen_rtx_XOR (<MODE>mode, copy_rtx (operands[1]),
+			     gen_lowpart (<MODE>mode, operands[2]));
+operands[10]
+= gen_rtx_COMPARE (GET_MODE (operands[8]),
+		       copy_rtx (operands[9]),
+		       const0_rtx);
+})
+;; Attempt to optimize away memory stores of values the memory already
+;; has.  See PR79593.
+(define_peephole2
+[(set (match_operand 0 "register_operand")
+(match_operand 1 "memory_operand"))
+(set (match_operand 2 "memory_operand") (match_dup 0))]
+"!MEM_VOLATILE_P (operands[1])
+&& !MEM_VOLATILE_P (operands[2])
+&& rtx_equal_p (operands[1], operands[2])
+&& !reg_overlap_mentioned_p (operands[0], operands[2])"
+[(set (match_dup 0) (match_dup 1))])
 ;; Attempt to always use XOR for zeroing registers (including FP modes).
 (define_peephole2
 [(set (match_operand 0 "general_reg_operand")
 	(match_operand 1 "const0_operand"))]
 [(prefetch (match_operand 0 "address_operand")
 	     (match_operand:SI 1 "const_int_operand")
 	     (match_operand:SI 2 "const_int_operand"))]
 "TARGET_3DNOW || TARGET_PREFETCH_SSE || TARGET_PRFCHW || TARGET_PREFETCHWT1"
 {
-bool write = INTVAL (operands[1]) != 0;
+bool write = operands[1] != const0_rtx;
 int locality = INTVAL (operands[2]);
 gcc_assert (IN_RANGE (locality, 0, 3));
 /* Use 3dNOW prefetch in case we are asking for write prefetch not
 [(prefetch (match_operand 0 "address_operand" "p")
 	     (match_operand:SI 1 "const_int_operand" "n")
 	     (const_int 3))]
 "TARGET_3DNOW || TARGET_PRFCHW || TARGET_PREFETCHWT1"
 {
-if (INTVAL (operands[1]) == 0)
+if (operands[1] == const0_rtx)
 return "prefetch\t%a0";
 else
 return "prefetchw\t%a0";
 }
 [(set_attr "type" "mmx")
 (set_attr "memory" "none")])
 (define_expand "stack_protect_set"
 [(match_operand 0 "memory_operand")
 (match_operand 1 "memory_operand")]
-"TARGET_SSP_TLS_GUARD"
+""
 {
-rtx (*insn)(rtx, rtx);
+emit_insn (gen_stack_protect_set_1
+	     (ptr_mode, operands[0], operands[1]));
-insn = (TARGET_LP64
-	  ? gen_stack_protect_set_di
-	  : gen_stack_protect_set_si);
-emit_insn (insn (operands[0], operands[1]));
 DONE;
 })
-(define_insn "stack_protect_set_<mode>"
+(define_insn "@stack_protect_set_1_<mode>"
 [(set (match_operand:PTR 0 "memory_operand" "=m")
 	(unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
 		    UNSPEC_SP_SET))
 (set (match_scratch:PTR 2 "=&r") (const_int 0))
 (clobber (reg:CC FLAGS_REG))]
-"TARGET_SSP_TLS_GUARD"
+""
-"mov{<imodesuffix>}\t{%1, %2|%2, %1}\;mov{<imodesuffix>}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2"
+{
+output_asm_insn ("mov{<imodesuffix>}\t{%1, %2|%2, %1}", operands);
+output_asm_insn ("mov{<imodesuffix>}\t{%2, %0|%0, %2}", operands);
+return "xor{l}\t%k2, %k2";
+}
 [(set_attr "type" "multi")])
+;; Patterns and peephole2s to optimize stack_protect_set_1_<mode>
+;; immediately followed by *mov{s,d}i_internal to the same register,
+;; where we can avoid the xor{l} above.  We don't split this, so that
+;; scheduling or anything else doesn't separate the *stack_protect_set*
+;; pattern from the set of the register that overwrites the register
+;; with a new value.
+(define_insn "*stack_protect_set_2_<mode>"
+[(set (match_operand:PTR 0 "memory_operand" "=m")
+	(unspec:PTR [(match_operand:PTR 3 "memory_operand" "m")]
+		    UNSPEC_SP_SET))
+(set (match_operand:SI 1 "register_operand" "=&r")
+	(match_operand:SI 2 "general_operand" "g"))
+(clobber (reg:CC FLAGS_REG))]
+"reload_completed
+&& !reg_overlap_mentioned_p (operands[1], operands[2])"
+{
+output_asm_insn ("mov{<imodesuffix>}\t{%3, %<k>1|%<k>1, %3}", operands);
+output_asm_insn ("mov{<imodesuffix>}\t{%<k>1, %0|%0, %<k>1}", operands);
+if (pic_32bit_operand (operands[2], SImode)
+|| ix86_use_lea_for_mov (insn, operands + 1))
+return "lea{l}\t{%E2, %1|%1, %E2}";
+else
+return "mov{l}\t{%2, %1|%1, %2}";
+}
+[(set_attr "type" "multi")
+(set_attr "length" "24")])
+(define_peephole2
+[(parallel [(set (match_operand:PTR 0 "memory_operand")
+		  (unspec:PTR [(match_operand:PTR 1 "memory_operand")]
+			      UNSPEC_SP_SET))
+	     (set (match_operand:PTR 2 "general_reg_operand") (const_int 0))
+	     (clobber (reg:CC FLAGS_REG))])
+(set (match_operand:SI 3 "general_reg_operand")
+(match_operand:SI 4))]
+"REGNO (operands[2]) == REGNO (operands[3])
+&& general_operand (operands[4], SImode)
+&& (general_reg_operand (operands[4], SImode)
+|| memory_operand (operands[4], SImode)
+|| immediate_operand (operands[4], SImode))
+&& !reg_overlap_mentioned_p (operands[3], operands[4])"
+[(parallel [(set (match_dup 0)
+		  (unspec:PTR [(match_dup 1)] UNSPEC_SP_SET))
+	     (set (match_dup 3) (match_dup 4))
+	     (clobber (reg:CC FLAGS_REG))])])
+(define_insn "*stack_protect_set_3"
+[(set (match_operand:DI 0 "memory_operand" "=m,m,m")
+	(unspec:DI [(match_operand:DI 3 "memory_operand" "m,m,m")]
+		   UNSPEC_SP_SET))
+(set (match_operand:DI 1 "register_operand" "=&r,r,r")
+	(match_operand:DI 2 "general_operand" "Z,rem,i"))
+(clobber (reg:CC FLAGS_REG))]
+"TARGET_64BIT
+&& reload_completed
+&& !reg_overlap_mentioned_p (operands[1], operands[2])"
+{
+output_asm_insn ("mov{q}\t{%3, %1|%1, %3}", operands);
+output_asm_insn ("mov{q}\t{%1, %0|%0, %1}", operands);
+if (pic_32bit_operand (operands[2], DImode))
+return "lea{q}\t{%E2, %1|%1, %E2}";
+else if (which_alternative == 0)
+return "mov{l}\t{%k2, %k1|%k1, %k2}";
+else if (which_alternative == 2)
+return "movabs{q}\t{%2, %1|%1, %2}";
+else if (ix86_use_lea_for_mov (insn, operands + 1))
+return "lea{q}\t{%E2, %1|%1, %E2}";
+else
+return "mov{q}\t{%2, %1|%1, %2}";
+}
+[(set_attr "type" "multi")
+(set_attr "length" "24")])
+(define_peephole2
+[(parallel [(set (match_operand:DI 0 "memory_operand")
+		  (unspec:DI [(match_operand:DI 1 "memory_operand")]
+			     UNSPEC_SP_SET))
+	     (set (match_operand:DI 2 "general_reg_operand") (const_int 0))
+	     (clobber (reg:CC FLAGS_REG))])
+(set (match_dup 2) (match_operand:DI 3))]
+"TARGET_64BIT
+&& general_operand (operands[3], DImode)
+&& (general_reg_operand (operands[3], DImode)
+|| memory_operand (operands[3], DImode)
+|| x86_64_zext_immediate_operand (operands[3], DImode)
+|| x86_64_immediate_operand (operands[3], DImode)
+|| (CONSTANT_P (operands[3])
+	  && (!flag_pic || LEGITIMATE_PIC_OPERAND_P (operands[3]))))
+&& !reg_overlap_mentioned_p (operands[2], operands[3])"
+[(parallel [(set (match_dup 0)
+		  (unspec:PTR [(match_dup 1)] UNSPEC_SP_SET))
+	     (set (match_dup 2) (match_dup 3))
+	     (clobber (reg:CC FLAGS_REG))])])
 (define_expand "stack_protect_test"
 [(match_operand 0 "memory_operand")
 (match_operand 1 "memory_operand")
 (match_operand 2)]
-"TARGET_SSP_TLS_GUARD"
+""
 {
 rtx flags = gen_rtx_REG (CCZmode, FLAGS_REG);
-rtx (*insn)(rtx, rtx, rtx);
+emit_insn (gen_stack_protect_test_1
+	     (ptr_mode, flags, operands[0], operands[1]));
-insn = (TARGET_LP64
-	  ? gen_stack_protect_test_di
-	  : gen_stack_protect_test_si);
-emit_insn (insn (flags, operands[0], operands[1]));
 emit_jump_insn (gen_cbranchcc4 (gen_rtx_EQ (VOIDmode, flags, const0_rtx),
 				  flags, const0_rtx, operands[2]));
 DONE;
 })
-(define_insn "stack_protect_test_<mode>"
+(define_insn "@stack_protect_test_1_<mode>"
 [(set (match_operand:CCZ 0 "flags_reg_operand")
 	(unspec:CCZ [(match_operand:PTR 1 "memory_operand" "m")
 		     (match_operand:PTR 2 "memory_operand" "m")]
 		    UNSPEC_SP_TEST))
 (clobber (match_scratch:PTR 3 "=&r"))]
-"TARGET_SSP_TLS_GUARD"
+""
-"mov{<imodesuffix>}\t{%1, %3|%3, %1}\;xor{<imodesuffix>}\t{%2, %3|%3, %2}"
+{
+output_asm_insn ("mov{<imodesuffix>}\t{%1, %3|%3, %1}", operands);
+return "sub{<imodesuffix>}\t{%2, %3|%3, %2}";
+}
 [(set_attr "type" "multi")])
 (define_insn "sse4_2_crc32<mode>"
 [(set (match_operand:SI 0 "register_operand" "=r")
 	(unspec:SI
 (define_expand "lwp_llwpcb"
 [(unspec_volatile [(match_operand 0 "register_operand")]
 		    UNSPECV_LLWP_INTRINSIC)]
 "TARGET_LWP")
-(define_insn "*lwp_llwpcb<mode>1"
+(define_insn "*lwp_llwpcb<mode>_1"
 [(unspec_volatile [(match_operand:P 0 "register_operand" "r")]
 		    UNSPECV_LLWP_INTRINSIC)]
 "TARGET_LWP"
 "llwpcb\t%0"
 [(set_attr "type" "lwp")
 (define_expand "lwp_slwpcb"
 [(set (match_operand 0 "register_operand")
 	(unspec_volatile [(const_int 0)] UNSPECV_SLWP_INTRINSIC))]
 "TARGET_LWP"
-{
+"emit_insn (gen_lwp_slwpcb_1 (Pmode, operands[0])); DONE;")
-rtx (*insn)(rtx);
+(define_insn "@lwp_slwpcb<mode>_1"
-insn = (Pmode == DImode
-	  ? gen_lwp_slwpcbdi
-	  : gen_lwp_slwpcbsi);
-emit_insn (insn (operands[0]));
-DONE;
-})
-(define_insn "lwp_slwpcb<mode>"
 [(set (match_operand:P 0 "register_operand" "=r")
 	(unspec_volatile:P [(const_int 0)] UNSPECV_SLWP_INTRINSIC))]
 "TARGET_LWP"
 "slwpcb\t%0"
 [(set_attr "type" "lwp")
 "TARGET_64BIT && TARGET_FSGSBASE"
 "wr<fsgs>base\t%0"
 [(set_attr "type" "other")
 (set_attr "prefix_extra" "2")])
+(define_insn "ptwrite<mode>"
+[(unspec_volatile [(match_operand:SWI48 0 "nonimmediate_operand" "rm")]
+		    UNSPECV_PTWRITE)]
+"TARGET_PTWRITE"
+"ptwrite\t%0"
+[(set_attr "type" "other")
+(set_attr "prefix_extra" "2")])
 (define_insn "rdrand<mode>_1"
 [(set (match_operand:SWI248 0 "register_operand" "=r")
 	(unspec_volatile:SWI248 [(const_int 0)] UNSPECV_RDRAND))
 (set (reg:CCC FLAGS_REG)
 	(unspec_volatile:CCC [(const_int 0)] UNSPECV_RDRAND))]
 ;; Since 32bit register operands are implicitly zero extended to 64bit,
 ;; we only need to set up 32bit registers.
 "mwaitx"
 [(set_attr "length" "3")])
-(define_insn "monitorx_<mode>"
+(define_insn "@monitorx_<mode>"
 [(unspec_volatile [(match_operand:P 0 "register_operand" "a")
 		     (match_operand:SI 1 "register_operand" "c")
 		     (match_operand:SI 2 "register_operand" "d")]
 		   UNSPECV_MONITORX)]
 "TARGET_MWAITX"
 "%^monitorx"
 [(set (attr "length")
 (symbol_ref ("(Pmode != word_mode) + 3")))])
 ;; CLZERO
-(define_insn "clzero_<mode>"
+(define_insn "@clzero_<mode>"
 [(unspec_volatile [(match_operand: P 0 "register_operand" "a")]
 UNSPECV_CLZERO)]
 "TARGET_CLZERO"
 "clzero"
 [(set_attr "length" "3")
 			  UNSPECV_MOVDIRI)]
 "TARGET_MOVDIRI"
 "movdiri\t{%1, %0|%0, %1}"
 [(set_attr "type" "other")])
-(define_insn "movdir64b_<mode>"
+(define_insn "@movdir64b_<mode>"
 [(unspec_volatile:XI [(match_operand:P 0 "register_operand" "r")
 			(match_operand:XI 1 "memory_operand")]
 		       UNSPECV_MOVDIR64B)]
 "TARGET_MOVDIR64B"
 "movdir64b\t{%1, %0|%0, %1}"
+[(set_attr "type" "other")])
+;; ENQCMD and ENQCMDS
+(define_int_iterator ENQCMD [UNSPECV_ENQCMD UNSPECV_ENQCMDS])
+(define_int_attr enqcmd_sfx [(UNSPECV_ENQCMD "") (UNSPECV_ENQCMDS "s")])
+(define_insn "@enqcmd<enqcmd_sfx>_<mode>"
+[(set (reg:CCZ FLAGS_REG)
+	(unspec_volatile:CCZ [(match_operand:P 0 "register_operand" "r")
+			      (match_operand:XI 1 "memory_operand" "m")]
+			     ENQCMD))]
+"TARGET_ENQCMD"
+"enqcmd<enqcmd_sfx>\t{%1, %0|%0, %1}"
 [(set_attr "type" "other")])
 ;; WAITPKG
 (define_insn "umwait"
 			     UNSPECV_UMWAIT))]
 "TARGET_64BIT && TARGET_WAITPKG"
 "umwait\t%0"
 [(set_attr "length" "3")])
-(define_insn "umonitor_<mode>"
+(define_insn "@umonitor_<mode>"
 [(unspec_volatile [(match_operand:P 0 "register_operand" "r")]
 		    UNSPECV_UMONITOR)]
 "TARGET_WAITPKG"
 "umonitor\t%0"
 [(set (attr "length")

Mercurial > hg > CbC > CbC_gcc

comparison gcc/config/i386/i386.md @ 146:351920fa3827