Mercurial > hg > CbC > CbC_gcc

comparison gcc/config/i386/sse.md @ 145:1830386684a0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
gcc-9.2.0
author anatofuz
date Thu, 13 Feb 2020 11:34:05 +0900
parents 84e7813d76e9
children
comparison
equal deleted inserted replaced
131:84e7813d76e9 145:1830386684a0
1 ;; GCC machine description for SSE instructions 1 ;; GCC machine description for SSE instructions
2 ;; Copyright (C) 2005-2018 Free Software Foundation, Inc. 2 ;; Copyright (C) 2005-2020 Free Software Foundation, Inc.
3 ;; 3 ;;
4 ;; This file is part of GCC. 4 ;; This file is part of GCC.
5 ;; 5 ;;
6 ;; GCC is free software; you can redistribute it and/or modify 6 ;; GCC is free software; you can redistribute it and/or modify
7 ;; it under the terms of the GNU General Public License as published by 7 ;; it under the terms of the GNU General Public License as published by
18 ;; <http://www.gnu.org/licenses/>. 18 ;; <http://www.gnu.org/licenses/>.
19 19
20 (define_c_enum "unspec" [ 20 (define_c_enum "unspec" [
21 ;; SSE 21 ;; SSE
22 UNSPEC_MOVNT 22 UNSPEC_MOVNT
23
24 ;; SSE2
25 UNSPEC_MOVDI_TO_SSE
23 26
24 ;; SSE3 27 ;; SSE3
25 UNSPEC_LDDQU 28 UNSPEC_LDDQU
26 29
27 ;; SSSE3 30 ;; SSSE3
182 ;; For VPCLMULQDQ support 185 ;; For VPCLMULQDQ support
183 UNSPEC_VPCLMULQDQ 186 UNSPEC_VPCLMULQDQ
184 187
185 ;; For AVX512BITALG support 188 ;; For AVX512BITALG support
186 UNSPEC_VPSHUFBIT 189 UNSPEC_VPSHUFBIT
190
191 ;; For VP2INTERSECT support
192 UNSPEC_VP2INTERSECT
193
194 ;; For AVX512BF16 support
195 UNSPEC_VCVTNE2PS2BF16
196 UNSPEC_VCVTNEPS2BF16
197 UNSPEC_VDPBF16PS
187 ]) 198 ])
188 199
189 (define_c_enum "unspecv" [ 200 (define_c_enum "unspecv" [
190 UNSPECV_LDMXCSR 201 UNSPECV_LDMXCSR
191 UNSPECV_STMXCSR 202 UNSPECV_STMXCSR
268 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")]) 279 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
269 280
270 ;; All SFmode vector float modes 281 ;; All SFmode vector float modes
271 (define_mode_iterator VF1 282 (define_mode_iterator VF1
272 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF]) 283 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF])
284
285 (define_mode_iterator VF1_AVX2
286 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX2") V4SF])
273 287
274 ;; 128- and 256-bit SF vector modes 288 ;; 128- and 256-bit SF vector modes
275 (define_mode_iterator VF1_128_256 289 (define_mode_iterator VF1_128_256
276 [(V8SF "TARGET_AVX") V4SF]) 290 [(V8SF "TARGET_AVX") V4SF])
277 291
582 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "V4SI") (V8DI "V2DI")]) 596 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "V4SI") (V8DI "V2DI")])
583 597
584 (define_mode_attr ssequarterinsnmode 598 (define_mode_attr ssequarterinsnmode
585 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "TI") (V8DI "TI")]) 599 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "TI") (V8DI "TI")])
586 600
601 (define_mode_attr vecmemsuffix
602 [(V16SF "{z}") (V8SF "{y}") (V4SF "{x}")
603 (V8DF "{z}") (V4DF "{y}") (V2DF "{x}")])
604
587 (define_mode_attr ssedoublemodelower 605 (define_mode_attr ssedoublemodelower
588 [(V16QI "v16hi") (V32QI "v32hi") (V64QI "v64hi") 606 [(V16QI "v16hi") (V32QI "v32hi") (V64QI "v64hi")
589 (V8HI "v8si") (V16HI "v16si") (V32HI "v32si") 607 (V8HI "v8si") (V16HI "v16si") (V32HI "v32si")
590 (V4SI "v4di") (V8SI "v8di") (V16SI "v16di")]) 608 (V4SI "v4di") (V8SI "v8di") (V16SI "v16di")])
591 609
592 (define_mode_attr ssedoublemode 610 (define_mode_attr ssedoublemode
593 [(V4SF "V8SF") (V8SF "V16SF") (V16SF "V32SF") 611 [(V4SF "V8SF") (V8SF "V16SF") (V16SF "V32SF")
594 (V2DF "V4DF") (V4DF "V8DF") (V8DF "V16DF") 612 (V2DF "V4DF") (V4DF "V8DF") (V8DF "V16DF")
595 (V16QI "V16HI") (V32QI "V32HI") (V64QI "V64HI") 613 (V16QI "V16HI") (V32QI "V32HI") (V64QI "V64HI")
596 (V4HI "V4SI") (V8HI "V8SI") (V16HI "V16SI") (V32HI "V32SI") 614 (V8HI "V8SI") (V16HI "V16SI") (V32HI "V32SI")
597 (V4SI "V4DI") (V8SI "V16SI") (V16SI "V32SI") 615 (V4SI "V4DI") (V8SI "V16SI") (V16SI "V32SI")
598 (V4DI "V8DI") (V8DI "V16DI")]) 616 (V4DI "V8DI") (V8DI "V16DI")])
599 617
600 (define_mode_attr ssebytemode 618 (define_mode_attr ssebytemode
601 [(V8DI "V64QI") (V4DI "V32QI") (V2DI "V16QI")]) 619 [(V8DI "V64QI") (V4DI "V32QI") (V2DI "V16QI")
620 (V16SI "V64QI") (V8SI "V32QI") (V4SI "V16QI")])
602 621
603 ;; All 128bit vector integer modes 622 ;; All 128bit vector integer modes
604 (define_mode_iterator VI_128 [V16QI V8HI V4SI V2DI]) 623 (define_mode_iterator VI_128 [V16QI V8HI V4SI V2DI])
605 624
606 ;; All 256bit vector integer modes 625 ;; All 256bit vector integer modes
720 (V16SI "hi") (V8SI "qi") (V4SI "qi") 739 (V16SI "hi") (V8SI "qi") (V4SI "qi")
721 (V8DI "qi") (V4DI "qi") (V2DI "qi") 740 (V8DI "qi") (V4DI "qi") (V2DI "qi")
722 (V16SF "hi") (V8SF "qi") (V4SF "qi") 741 (V16SF "hi") (V8SF "qi") (V4SF "qi")
723 (V8DF "qi") (V4DF "qi") (V2DF "qi")]) 742 (V8DF "qi") (V4DF "qi") (V2DF "qi")])
724 743
744 ;; Mapping of vector modes to corresponding mask half size
745 (define_mode_attr avx512fmaskhalfmode
746 [(V64QI "SI") (V32QI "HI") (V16QI "QI")
747 (V32HI "HI") (V16HI "QI") (V8HI "QI") (V4HI "QI")
748 (V16SI "QI") (V8SI "QI") (V4SI "QI")
749 (V8DI "QI") (V4DI "QI") (V2DI "QI")
750 (V16SF "QI") (V8SF "QI") (V4SF "QI")
751 (V8DF "QI") (V4DF "QI") (V2DF "QI")])
752
725 ;; Mapping of vector float modes to an integer mode of the same size 753 ;; Mapping of vector float modes to an integer mode of the same size
726 (define_mode_attr sseintvecmode 754 (define_mode_attr sseintvecmode
727 [(V16SF "V16SI") (V8DF "V8DI") 755 [(V16SF "V16SI") (V8DF "V8DI")
728 (V8SF "V8SI") (V4DF "V4DI") 756 (V8SF "V8SI") (V4DF "V4DI")
729 (V4SF "V4SI") (V2DF "V2DI") 757 (V4SF "V4SI") (V2DF "V2DI")
822 (V16QI "b") (V8HI "w") (V4SI "k") (V2DI "q") 850 (V16QI "b") (V8HI "w") (V4SI "k") (V2DI "q")
823 (V16SF "k") (V8DF "q") 851 (V16SF "k") (V8DF "q")
824 (V8SF "k") (V4DF "q") 852 (V8SF "k") (V4DF "q")
825 (V4SF "k") (V2DF "q") 853 (V4SF "k") (V2DF "q")
826 (SF "k") (DF "q")]) 854 (SF "k") (DF "q")])
855
856 ;; Mapping of vector modes to VPTERNLOG suffix
857 (define_mode_attr ternlogsuffix
858 [(V8DI "q") (V4DI "q") (V2DI "q")
859 (V16SI "d") (V8SI "d") (V4SI "d")
860 (V32HI "d") (V16HI "d") (V8HI "d")
861 (V64QI "d") (V32QI "d") (V16QI "d")])
827 862
828 ;; Number of scalar elements in each vector type 863 ;; Number of scalar elements in each vector type
829 (define_mode_attr ssescalarnum 864 (define_mode_attr ssescalarnum
830 [(V64QI "64") (V16SI "16") (V8DI "8") 865 [(V64QI "64") (V16SI "16") (V8DI "8")
831 (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4") 866 (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
1079 (set_attr "prefix" "maybe_vex") 1114 (set_attr "prefix" "maybe_vex")
1080 (set (attr "mode") 1115 (set (attr "mode")
1081 (cond [(and (eq_attr "alternative" "1") 1116 (cond [(and (eq_attr "alternative" "1")
1082 (match_test "TARGET_AVX512VL")) 1117 (match_test "TARGET_AVX512VL"))
1083 (const_string "<sseinsnmode>") 1118 (const_string "<sseinsnmode>")
1084 (and (match_test "<MODE_SIZE> == 16")
1085 (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1086 (and (eq_attr "alternative" "3")
1087 (match_test "TARGET_SSE_TYPELESS_STORES"))))
1088 (const_string "<ssePSmode>")
1089 (match_test "TARGET_AVX") 1119 (match_test "TARGET_AVX")
1090 (const_string "<sseinsnmode>") 1120 (const_string "<sseinsnmode>")
1091 (ior (not (match_test "TARGET_SSE2")) 1121 (ior (not (match_test "TARGET_SSE2"))
1092 (match_test "optimize_function_for_size_p (cfun)")) 1122 (match_test "optimize_function_for_size_p (cfun)"))
1123 (const_string "V4SF")
1124 (and (match_test "<MODE>mode == V2DFmode")
1125 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
1126 (const_string "V4SF")
1127 (and (eq_attr "alternative" "3")
1128 (match_test "TARGET_SSE_TYPELESS_STORES"))
1093 (const_string "V4SF") 1129 (const_string "V4SF")
1094 (and (eq_attr "alternative" "0") 1130 (and (eq_attr "alternative" "0")
1095 (match_test "TARGET_SSE_LOAD0_BY_PXOR")) 1131 (match_test "TARGET_SSE_LOAD0_BY_PXOR"))
1096 (const_string "TI") 1132 (const_string "TI")
1097 ] 1133 ]
1145 [(set_attr "type" "ssemov") 1181 [(set_attr "type" "ssemov")
1146 (set_attr "prefix" "evex") 1182 (set_attr "prefix" "evex")
1147 (set_attr "memory" "none,load") 1183 (set_attr "memory" "none,load")
1148 (set_attr "mode" "<sseinsnmode>")]) 1184 (set_attr "mode" "<sseinsnmode>")])
1149 1185
1186 (define_insn "avx512f_mov<ssescalarmodelower>_mask"
1187 [(set (match_operand:VF_128 0 "register_operand" "=v")
1188 (vec_merge:VF_128
1189 (vec_merge:VF_128
1190 (match_operand:VF_128 2 "register_operand" "v")
1191 (match_operand:VF_128 3 "nonimm_or_0_operand" "0C")
1192 (match_operand:QI 4 "register_operand" "Yk"))
1193 (match_operand:VF_128 1 "register_operand" "v")
1194 (const_int 1)))]
1195 "TARGET_AVX512F"
1196 "vmov<ssescalarmodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
1197 [(set_attr "type" "ssemov")
1198 (set_attr "prefix" "evex")
1199 (set_attr "mode" "<ssescalarmode>")])
1200
1201 (define_expand "avx512f_load<mode>_mask"
1202 [(set (match_operand:<ssevecmode> 0 "register_operand")
1203 (vec_merge:<ssevecmode>
1204 (vec_merge:<ssevecmode>
1205 (vec_duplicate:<ssevecmode>
1206 (match_operand:MODEF 1 "memory_operand"))
1207 (match_operand:<ssevecmode> 2 "nonimm_or_0_operand")
1208 (match_operand:QI 3 "register_operand"))
1209 (match_dup 4)
1210 (const_int 1)))]
1211 "TARGET_AVX512F"
1212 "operands[4] = CONST0_RTX (<ssevecmode>mode);")
1213
1214 (define_insn "*avx512f_load<mode>_mask"
1215 [(set (match_operand:<ssevecmode> 0 "register_operand" "=v")
1216 (vec_merge:<ssevecmode>
1217 (vec_merge:<ssevecmode>
1218 (vec_duplicate:<ssevecmode>
1219 (match_operand:MODEF 1 "memory_operand" "m"))
1220 (match_operand:<ssevecmode> 2 "nonimm_or_0_operand" "0C")
1221 (match_operand:QI 3 "register_operand" "Yk"))
1222 (match_operand:<ssevecmode> 4 "const0_operand" "C")
1223 (const_int 1)))]
1224 "TARGET_AVX512F"
1225 "vmov<ssescalarmodesuffix>\t{%1, %0%{%3%}%N2|%0%{3%}%N2, %1}"
1226 [(set_attr "type" "ssemov")
1227 (set_attr "prefix" "evex")
1228 (set_attr "memory" "load")
1229 (set_attr "mode" "<MODE>")])
1230
1231 (define_insn "avx512f_store<mode>_mask"
1232 [(set (match_operand:MODEF 0 "memory_operand" "=m")
1233 (if_then_else:MODEF
1234 (and:QI (match_operand:QI 2 "register_operand" "Yk")
1235 (const_int 1))
1236 (vec_select:MODEF
1237 (match_operand:<ssevecmode> 1 "register_operand" "v")
1238 (parallel [(const_int 0)]))
1239 (match_dup 0)))]
1240 "TARGET_AVX512F"
1241 "vmov<ssescalarmodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
1242 [(set_attr "type" "ssemov")
1243 (set_attr "prefix" "evex")
1244 (set_attr "memory" "store")
1245 (set_attr "mode" "<MODE>")])
1246
1150 (define_insn "<avx512>_blendm<mode>" 1247 (define_insn "<avx512>_blendm<mode>"
1151 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v") 1248 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v")
1152 (vec_merge:V48_AVX512VL 1249 (vec_merge:V48_AVX512VL
1153 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "vm") 1250 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "vm")
1154 (match_operand:V48_AVX512VL 1 "register_operand" "v") 1251 (match_operand:V48_AVX512VL 1 "register_operand" "v")
1233 ;; come from memory, this is the best we can do. This is much better 1330 ;; come from memory, this is the best we can do. This is much better
1234 ;; than storing %edx:%eax into a stack temporary and loading an %xmm 1331 ;; than storing %edx:%eax into a stack temporary and loading an %xmm
1235 ;; from there. 1332 ;; from there.
1236 1333
1237 (define_insn_and_split "movdi_to_sse" 1334 (define_insn_and_split "movdi_to_sse"
1238 [(parallel 1335 [(set (match_operand:V4SI 0 "register_operand" "=x,x,?x")
1239 [(set (match_operand:V4SI 0 "register_operand" "=?x,x") 1336 (unspec:V4SI [(match_operand:DI 1 "nonimmediate_operand" "r,m,r")]
1240 (subreg:V4SI (match_operand:DI 1 "nonimmediate_operand" "r,m") 0)) 1337 UNSPEC_MOVDI_TO_SSE))
1241 (clobber (match_scratch:V4SI 2 "=&x,X"))])] 1338 (clobber (match_scratch:V4SI 2 "=X,X,&x"))]
1242 "!TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_TO_VEC" 1339 "!TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_TO_VEC"
1243 "#" 1340 "#"
1244 "&& reload_completed" 1341 "&& reload_completed"
1245 [(const_int 0)] 1342 [(const_int 0)]
1246 { 1343 {
1248 { 1345 {
1249 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax). 1346 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
1250 Assemble the 64-bit DImode value in an xmm register. */ 1347 Assemble the 64-bit DImode value in an xmm register. */
1251 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode), 1348 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode),
1252 gen_lowpart (SImode, operands[1]))); 1349 gen_lowpart (SImode, operands[1])));
1253 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode), 1350 if (TARGET_SSE4_1)
1254 gen_highpart (SImode, operands[1]))); 1351 emit_insn (gen_sse4_1_pinsrd (operands[0], operands[0],
1255 emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0], 1352 gen_highpart (SImode, operands[1]),
1256 operands[2])); 1353 GEN_INT (2)));
1257 } 1354 else
1355 {
1356 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode),
1357 gen_highpart (SImode, operands[1])));
1358 emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
1359 operands[2]));
1360 }
1361 }
1258 else if (memory_operand (operands[1], DImode)) 1362 else if (memory_operand (operands[1], DImode))
1259 emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]), 1363 emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]),
1260 operands[1], const0_rtx)); 1364 operands[1], const0_rtx));
1261 else 1365 else
1262 gcc_unreachable (); 1366 gcc_unreachable ();
1263 DONE; 1367 DONE;
1264 }) 1368 }
1369 [(set_attr "isa" "sse4,*,*")])
1265 1370
1266 (define_split 1371 (define_split
1267 [(set (match_operand:V4SF 0 "register_operand") 1372 [(set (match_operand:V4SF 0 "register_operand")
1268 (match_operand:V4SF 1 "zero_extended_scalar_load_operand"))] 1373 (match_operand:V4SF 1 "zero_extended_scalar_load_operand"))]
1269 "TARGET_SSE && reload_completed" 1374 "TARGET_SSE && reload_completed"
1624 (absneg:VF 1729 (absneg:VF
1625 (match_operand:VF 1 "register_operand")))] 1730 (match_operand:VF 1 "register_operand")))]
1626 "TARGET_SSE" 1731 "TARGET_SSE"
1627 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;") 1732 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
1628 1733
1629 (define_insn_and_split "*absneg<mode>2" 1734 (define_insn_and_split "*<code><mode>2"
1630 [(set (match_operand:VF 0 "register_operand" "=x,x,v,v") 1735 [(set (match_operand:VF 0 "register_operand" "=x,x,v,v")
1631 (match_operator:VF 3 "absneg_operator" 1736 (absneg:VF
1632 [(match_operand:VF 1 "vector_operand" "0, xBm,v, m")])) 1737 (match_operand:VF 1 "vector_operand" "0, xBm,v, m")))
1633 (use (match_operand:VF 2 "vector_operand" "xBm,0, vm,v"))] 1738 (use (match_operand:VF 2 "vector_operand" "xBm,0, vm,v"))]
1634 "TARGET_SSE" 1739 "TARGET_SSE"
1635 "#" 1740 "#"
1636 "&& reload_completed" 1741 "&& reload_completed"
1637 [(const_int 0)] 1742 [(set (match_dup 0) (match_dup 3))]
1638 { 1743 {
1639 enum rtx_code absneg_op; 1744 enum rtx_code absneg_op = <CODE> == ABS ? AND : XOR;
1640 rtx op1, op2;
1641 rtx t;
1642 1745
1643 if (TARGET_AVX) 1746 if (TARGET_AVX)
1644 { 1747 {
1645 if (MEM_P (operands[1])) 1748 if (MEM_P (operands[1]))
1646 op1 = operands[2], op2 = operands[1]; 1749 std::swap (operands[1], operands[2]);
1647 else
1648 op1 = operands[1], op2 = operands[2];
1649 } 1750 }
1650 else 1751 else
1752 {
1753 if (operands_match_p (operands[0], operands[2]))
1754 std::swap (operands[1], operands[2]);
1755 }
1756
1757 operands[3]
1758 = gen_rtx_fmt_ee (absneg_op, <MODE>mode, operands[1], operands[2]);
1759 }
1760 [(set_attr "isa" "noavx,noavx,avx,avx")])
1761
1762 (define_insn_and_split "*nabs<mode>2"
1763 [(set (match_operand:VF 0 "register_operand" "=x,x,v,v")
1764 (neg:VF
1765 (abs:VF
1766 (match_operand:VF 1 "vector_operand" "0,xBm,v,m"))))
1767 (use (match_operand:VF 2 "vector_operand" "xBm,0,vm,v"))]
1768 "TARGET_SSE"
1769 "#"
1770 "&& reload_completed"
1771 [(set (match_dup 0) (match_dup 3))]
1772 {
1773 if (TARGET_AVX)
1651 { 1774 {
1652 op1 = operands[0]; 1775 if (MEM_P (operands[1]))
1653 if (rtx_equal_p (operands[0], operands[1])) 1776 std::swap (operands[1], operands[2]);
1654 op2 = operands[2];
1655 else
1656 op2 = operands[1];
1657 } 1777 }
1658 1778 else
1659 absneg_op = GET_CODE (operands[3]) == NEG ? XOR : AND; 1779 {
1660 t = gen_rtx_fmt_ee (absneg_op, <MODE>mode, op1, op2); 1780 if (operands_match_p (operands[0], operands[2]))
1661 t = gen_rtx_SET (operands[0], t); 1781 std::swap (operands[1], operands[2]);
1662 emit_insn (t); 1782 }
1663 DONE; 1783
1784 operands[3]
1785 = gen_rtx_fmt_ee (IOR, <MODE>mode, operands[1], operands[2]);
1664 } 1786 }
1665 [(set_attr "isa" "noavx,noavx,avx,avx")]) 1787 [(set_attr "isa" "noavx,noavx,avx,avx")])
1666 1788
1667 (define_expand "<plusminus_insn><mode>3<mask_name><round_name>" 1789 (define_expand "<plusminus_insn><mode>3<mask_name><round_name>"
1668 [(set (match_operand:VF 0 "register_operand") 1790 [(set (match_operand:VF 0 "register_operand")
1713 "vadd<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}" 1835 "vadd<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
1714 [(set_attr "prefix" "evex") 1836 [(set_attr "prefix" "evex")
1715 (set_attr "type" "sseadd") 1837 (set_attr "type" "sseadd")
1716 (set_attr "mode" "<MODE>")]) 1838 (set_attr "mode" "<MODE>")])
1717 1839
1840 ;; Standard scalar operation patterns which preserve the rest of the
1841 ;; vector for combiner.
1842 (define_insn "*<sse>_vm<plusminus_insn><mode>3"
1843 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1844 (vec_merge:VF_128
1845 (vec_duplicate:VF_128
1846 (plusminus:<ssescalarmode>
1847 (vec_select:<ssescalarmode>
1848 (match_operand:VF_128 1 "register_operand" "0,v")
1849 (parallel [(const_int 0)]))
1850 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "xm,vm")))
1851 (match_dup 1)
1852 (const_int 1)))]
1853 "TARGET_SSE"
1854 "@
1855 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %2}
1856 v<plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1857 [(set_attr "isa" "noavx,avx")
1858 (set_attr "type" "sseadd")
1859 (set_attr "prefix" "orig,vex")
1860 (set_attr "mode" "<ssescalarmode>")])
1861
1718 (define_insn "<sse>_vm<plusminus_insn><mode>3<mask_scalar_name><round_scalar_name>" 1862 (define_insn "<sse>_vm<plusminus_insn><mode>3<mask_scalar_name><round_scalar_name>"
1719 [(set (match_operand:VF_128 0 "register_operand" "=x,v") 1863 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1720 (vec_merge:VF_128 1864 (vec_merge:VF_128
1721 (plusminus:VF_128 1865 (plusminus:VF_128
1722 (match_operand:VF_128 1 "register_operand" "0,v") 1866 (match_operand:VF_128 1 "register_operand" "0,v")
1723 (match_operand:VF_128 2 "vector_operand" "xBm,<round_scalar_constraint>")) 1867 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_scalar_constraint>"))
1724 (match_dup 1) 1868 (match_dup 1)
1725 (const_int 1)))] 1869 (const_int 1)))]
1726 "TARGET_SSE" 1870 "TARGET_SSE"
1727 "@ 1871 "@
1728 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2} 1872 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1767 "vmul<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<<avx512bcst>>}" 1911 "vmul<ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<<avx512bcst>>}"
1768 [(set_attr "prefix" "evex") 1912 [(set_attr "prefix" "evex")
1769 (set_attr "type" "ssemul") 1913 (set_attr "type" "ssemul")
1770 (set_attr "mode" "<MODE>")]) 1914 (set_attr "mode" "<MODE>")])
1771 1915
1916 ;; Standard scalar operation patterns which preserve the rest of the
1917 ;; vector for combiner.
1918 (define_insn "*<sse>_vm<multdiv_mnemonic><mode>3"
1919 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1920 (vec_merge:VF_128
1921 (vec_duplicate:VF_128
1922 (multdiv:<ssescalarmode>
1923 (vec_select:<ssescalarmode>
1924 (match_operand:VF_128 1 "register_operand" "0,v")
1925 (parallel [(const_int 0)]))
1926 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "xm,vm")))
1927 (match_dup 1)
1928 (const_int 1)))]
1929 "TARGET_SSE"
1930 "@
1931 <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %2}
1932 v<multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1933 [(set_attr "isa" "noavx,avx")
1934 (set_attr "type" "sse<multdiv_mnemonic>")
1935 (set_attr "prefix" "orig,vex")
1936 (set_attr "btver2_decode" "direct,double")
1937 (set_attr "mode" "<ssescalarmode>")])
1938
1772 (define_insn "<sse>_vm<multdiv_mnemonic><mode>3<mask_scalar_name><round_scalar_name>" 1939 (define_insn "<sse>_vm<multdiv_mnemonic><mode>3<mask_scalar_name><round_scalar_name>"
1773 [(set (match_operand:VF_128 0 "register_operand" "=x,v") 1940 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1774 (vec_merge:VF_128 1941 (vec_merge:VF_128
1775 (multdiv:VF_128 1942 (multdiv:VF_128
1776 (match_operand:VF_128 1 "register_operand" "0,v") 1943 (match_operand:VF_128 1 "register_operand" "0,v")
1777 (match_operand:VF_128 2 "vector_operand" "xBm,<round_scalar_constraint>")) 1944 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_scalar_constraint>"))
1778 (match_dup 1) 1945 (match_dup 1)
1779 (const_int 1)))] 1946 (const_int 1)))]
1780 "TARGET_SSE" 1947 "TARGET_SSE"
1781 "@ 1948 "@
1782 <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2} 1949 <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1867 (set_attr "atom_sse_attr" "rcp") 2034 (set_attr "atom_sse_attr" "rcp")
1868 (set_attr "btver2_sse_attr" "rcp") 2035 (set_attr "btver2_sse_attr" "rcp")
1869 (set_attr "prefix" "orig,vex") 2036 (set_attr "prefix" "orig,vex")
1870 (set_attr "mode" "SF")]) 2037 (set_attr "mode" "SF")])
1871 2038
2039 (define_insn "*sse_vmrcpv4sf2"
2040 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2041 (vec_merge:V4SF
2042 (vec_duplicate:V4SF
2043 (unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "xm,xm")]
2044 UNSPEC_RCP))
2045 (match_operand:V4SF 2 "register_operand" "0,x")
2046 (const_int 1)))]
2047 "TARGET_SSE"
2048 "@
2049 rcpss\t{%1, %0|%0, %1}
2050 vrcpss\t{%1, %2, %0|%0, %2, %1}"
2051 [(set_attr "isa" "noavx,avx")
2052 (set_attr "type" "sse")
2053 (set_attr "atom_sse_attr" "rcp")
2054 (set_attr "btver2_sse_attr" "rcp")
2055 (set_attr "prefix" "orig,vex")
2056 (set_attr "mode" "SF")])
2057
1872 (define_insn "<mask_codefor>rcp14<mode><mask_name>" 2058 (define_insn "<mask_codefor>rcp14<mode><mask_name>"
1873 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 2059 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
1874 (unspec:VF_AVX512VL 2060 (unspec:VF_AVX512VL
1875 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")] 2061 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")]
1876 UNSPEC_RCP14))] 2062 UNSPEC_RCP14))]
1948 2134
1949 (define_insn "<sse>_vmsqrt<mode>2<mask_scalar_name><round_scalar_name>" 2135 (define_insn "<sse>_vmsqrt<mode>2<mask_scalar_name><round_scalar_name>"
1950 [(set (match_operand:VF_128 0 "register_operand" "=x,v") 2136 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1951 (vec_merge:VF_128 2137 (vec_merge:VF_128
1952 (sqrt:VF_128 2138 (sqrt:VF_128
1953 (match_operand:VF_128 1 "vector_operand" "xBm,<round_scalar_constraint>")) 2139 (match_operand:VF_128 1 "nonimmediate_operand" "xm,<round_scalar_constraint>"))
1954 (match_operand:VF_128 2 "register_operand" "0,v") 2140 (match_operand:VF_128 2 "register_operand" "0,v")
1955 (const_int 1)))] 2141 (const_int 1)))]
1956 "TARGET_SSE" 2142 "TARGET_SSE"
1957 "@ 2143 "@
1958 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1} 2144 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1}
1962 (set_attr "atom_sse_attr" "sqrt") 2148 (set_attr "atom_sse_attr" "sqrt")
1963 (set_attr "prefix" "<round_scalar_prefix>") 2149 (set_attr "prefix" "<round_scalar_prefix>")
1964 (set_attr "btver2_sse_attr" "sqrt") 2150 (set_attr "btver2_sse_attr" "sqrt")
1965 (set_attr "mode" "<ssescalarmode>")]) 2151 (set_attr "mode" "<ssescalarmode>")])
1966 2152
2153 (define_insn "*<sse>_vmsqrt<mode>2<mask_scalar_name><round_scalar_name>"
2154 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
2155 (vec_merge:VF_128
2156 (vec_duplicate:VF_128
2157 (sqrt:<ssescalarmode>
2158 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "xm,<round_scalar_constraint>")))
2159 (match_operand:VF_128 2 "register_operand" "0,v")
2160 (const_int 1)))]
2161 "TARGET_SSE"
2162 "@
2163 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %1}
2164 vsqrt<ssescalarmodesuffix>\t{<round_scalar_mask_op3>%1, %2, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %2, %1<round_scalar_mask_op3>}"
2165 [(set_attr "isa" "noavx,avx")
2166 (set_attr "type" "sse")
2167 (set_attr "atom_sse_attr" "sqrt")
2168 (set_attr "prefix" "<round_scalar_prefix>")
2169 (set_attr "btver2_sse_attr" "sqrt")
2170 (set_attr "mode" "<ssescalarmode>")])
2171
1967 (define_expand "rsqrt<mode>2" 2172 (define_expand "rsqrt<mode>2"
1968 [(set (match_operand:VF1_128_256 0 "register_operand") 2173 [(set (match_operand:VF1_128_256 0 "register_operand")
1969 (unspec:VF1_128_256 2174 (unspec:VF1_128_256
1970 [(match_operand:VF1_128_256 1 "vector_operand")] UNSPEC_RSQRT))] 2175 [(match_operand:VF1_128_256 1 "vector_operand")] UNSPEC_RSQRT))]
1971 "TARGET_SSE_MATH" 2176 "TARGET_SSE && TARGET_SSE_MATH"
1972 { 2177 {
1973 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true); 2178 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true);
1974 DONE; 2179 DONE;
1975 }) 2180 })
1976 2181
1977 (define_expand "rsqrtv16sf2" 2182 (define_expand "rsqrtv16sf2"
1978 [(set (match_operand:V16SF 0 "register_operand") 2183 [(set (match_operand:V16SF 0 "register_operand")
1979 (unspec:V16SF 2184 (unspec:V16SF
1980 [(match_operand:V16SF 1 "vector_operand")] 2185 [(match_operand:V16SF 1 "vector_operand")]
1981 UNSPEC_RSQRT28))] 2186 UNSPEC_RSQRT28))]
1982 "TARGET_SSE_MATH && TARGET_AVX512ER" 2187 "TARGET_AVX512ER && TARGET_SSE_MATH"
1983 { 2188 {
1984 ix86_emit_swsqrtsf (operands[0], operands[1], V16SFmode, true); 2189 ix86_emit_swsqrtsf (operands[0], operands[1], V16SFmode, true);
1985 DONE; 2190 DONE;
1986 }) 2191 })
1987 2192
2046 (const_int 1)))] 2251 (const_int 1)))]
2047 "TARGET_SSE" 2252 "TARGET_SSE"
2048 "@ 2253 "@
2049 rsqrtss\t{%1, %0|%0, %k1} 2254 rsqrtss\t{%1, %0|%0, %k1}
2050 vrsqrtss\t{%1, %2, %0|%0, %2, %k1}" 2255 vrsqrtss\t{%1, %2, %0|%0, %2, %k1}"
2256 [(set_attr "isa" "noavx,avx")
2257 (set_attr "type" "sse")
2258 (set_attr "prefix" "orig,vex")
2259 (set_attr "mode" "SF")])
2260
2261 (define_insn "*sse_vmrsqrtv4sf2"
2262 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2263 (vec_merge:V4SF
2264 (vec_duplicate:V4SF
2265 (unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "xm,xm")]
2266 UNSPEC_RSQRT))
2267 (match_operand:V4SF 2 "register_operand" "0,x")
2268 (const_int 1)))]
2269 "TARGET_SSE"
2270 "@
2271 rsqrtss\t{%1, %0|%0, %1}
2272 vrsqrtss\t{%1, %2, %0|%0, %2, %1}"
2051 [(set_attr "isa" "noavx,avx") 2273 [(set_attr "isa" "noavx,avx")
2052 (set_attr "type" "sse") 2274 (set_attr "type" "sse")
2053 (set_attr "prefix" "orig,vex") 2275 (set_attr "prefix" "orig,vex")
2054 (set_attr "mode" "SF")]) 2276 (set_attr "mode" "SF")])
2055 2277
2116 (set_attr "type" "sseadd") 2338 (set_attr "type" "sseadd")
2117 (set_attr "btver2_sse_attr" "maxmin") 2339 (set_attr "btver2_sse_attr" "maxmin")
2118 (set_attr "prefix" "<mask_prefix3>") 2340 (set_attr "prefix" "<mask_prefix3>")
2119 (set_attr "mode" "<MODE>")]) 2341 (set_attr "mode" "<MODE>")])
2120 2342
2343 ;; Standard scalar operation patterns which preserve the rest of the
2344 ;; vector for combiner.
2345 (define_insn "*ieee_<ieee_maxmin><mode>3"
2346 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
2347 (vec_merge:VF_128
2348 (vec_duplicate:VF_128
2349 (unspec:<ssescalarmode>
2350 [(vec_select:<ssescalarmode>
2351 (match_operand:VF_128 1 "register_operand" "0,v")
2352 (parallel [(const_int 0)]))
2353 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "xm,vm")]
2354 IEEE_MAXMIN))
2355 (match_dup 1)
2356 (const_int 1)))]
2357 "TARGET_SSE"
2358 "@
2359 <ieee_maxmin><ssescalarmodesuffix>\t{%2, %0|%0, %2}
2360 v<ieee_maxmin><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2361 [(set_attr "isa" "noavx,avx")
2362 (set_attr "type" "sseadd")
2363 (set_attr "btver2_sse_attr" "maxmin")
2364 (set_attr "prefix" "orig,vex")
2365 (set_attr "mode" "<ssescalarmode>")])
2366
2121 (define_insn "<sse>_vm<code><mode>3<mask_scalar_name><round_saeonly_scalar_name>" 2367 (define_insn "<sse>_vm<code><mode>3<mask_scalar_name><round_saeonly_scalar_name>"
2122 [(set (match_operand:VF_128 0 "register_operand" "=x,v") 2368 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
2123 (vec_merge:VF_128 2369 (vec_merge:VF_128
2124 (smaxmin:VF_128 2370 (smaxmin:VF_128
2125 (match_operand:VF_128 1 "register_operand" "0,v") 2371 (match_operand:VF_128 1 "register_operand" "0,v")
2126 (match_operand:VF_128 2 "vector_operand" "xBm,<round_saeonly_scalar_constraint>")) 2372 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_saeonly_scalar_constraint>"))
2127 (match_dup 1) 2373 (match_dup 1)
2128 (const_int 1)))] 2374 (const_int 1)))]
2129 "TARGET_SSE" 2375 "TARGET_SSE"
2130 "@ 2376 "@
2131 <maxmin_float><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2} 2377 <maxmin_float><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
2536 emit_insn (gen_vec_extract<mode><ssescalarmodelower> (operands[0], tmp, 2782 emit_insn (gen_vec_extract<mode><ssescalarmodelower> (operands[0], tmp,
2537 const0_rtx)); 2783 const0_rtx));
2538 DONE; 2784 DONE;
2539 }) 2785 })
2540 2786
2787 (define_expand "reduc_plus_scal_v16qi"
2788 [(plus:V16QI
2789 (match_operand:QI 0 "register_operand")
2790 (match_operand:V16QI 1 "register_operand"))]
2791 "TARGET_SSE2"
2792 {
2793 rtx tmp = gen_reg_rtx (V1TImode);
2794 emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, operands[1]),
2795 GEN_INT (64)));
2796 rtx tmp2 = gen_reg_rtx (V16QImode);
2797 emit_insn (gen_addv16qi3 (tmp2, operands[1], gen_lowpart (V16QImode, tmp)));
2798 rtx tmp3 = gen_reg_rtx (V16QImode);
2799 emit_move_insn (tmp3, CONST0_RTX (V16QImode));
2800 rtx tmp4 = gen_reg_rtx (V2DImode);
2801 emit_insn (gen_sse2_psadbw (tmp4, tmp2, tmp3));
2802 tmp4 = gen_lowpart (V16QImode, tmp4);
2803 emit_insn (gen_vec_extractv16qiqi (operands[0], tmp4, const0_rtx));
2804 DONE;
2805 })
2806
2541 (define_mode_iterator REDUC_PLUS_MODE 2807 (define_mode_iterator REDUC_PLUS_MODE
2542 [(V4DF "TARGET_AVX") (V8SF "TARGET_AVX") 2808 [(V4DF "TARGET_AVX") (V8SF "TARGET_AVX")
2543 (V8DF "TARGET_AVX512F") (V16SF "TARGET_AVX512F")]) 2809 (V8DF "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
2810 (V32QI "TARGET_AVX") (V64QI "TARGET_AVX512F")])
2544 2811
2545 (define_expand "reduc_plus_scal_<mode>" 2812 (define_expand "reduc_plus_scal_<mode>"
2546 [(plus:REDUC_PLUS_MODE 2813 [(plus:REDUC_PLUS_MODE
2547 (match_operand:<ssescalarmode> 0 "register_operand") 2814 (match_operand:<ssescalarmode> 0 "register_operand")
2548 (match_operand:REDUC_PLUS_MODE 1 "register_operand"))] 2815 (match_operand:REDUC_PLUS_MODE 1 "register_operand"))]
2549 "" 2816 ""
2550 { 2817 {
2551 rtx tmp = gen_reg_rtx (<ssehalfvecmode>mode); 2818 rtx tmp = gen_reg_rtx (<ssehalfvecmode>mode);
2552 emit_insn (gen_vec_extract_hi_<mode> (tmp, operands[1])); 2819 emit_insn (gen_vec_extract_hi_<mode> (tmp, operands[1]));
2553 rtx tmp2 = gen_reg_rtx (<ssehalfvecmode>mode); 2820 rtx tmp2 = gen_reg_rtx (<ssehalfvecmode>mode);
2554 emit_insn (gen_add<ssehalfvecmodelower>3 2821 rtx tmp3 = gen_lowpart (<ssehalfvecmode>mode, operands[1]);
2555 (tmp2, tmp, gen_lowpart (<ssehalfvecmode>mode, operands[1]))); 2822 emit_insn (gen_add<ssehalfvecmodelower>3 (tmp2, tmp, tmp3));
2556 emit_insn (gen_reduc_plus_scal_<ssehalfvecmodelower> (operands[0], tmp2)); 2823 emit_insn (gen_reduc_plus_scal_<ssehalfvecmodelower> (operands[0], tmp2));
2557 DONE; 2824 DONE;
2558 }) 2825 })
2559 2826
2560 ;; Modes handled by reduc_sm{in,ax}* patterns. 2827 ;; Modes handled by reduc_sm{in,ax}* patterns.
2561 (define_mode_iterator REDUC_SSE_SMINMAX_MODE 2828 (define_mode_iterator REDUC_SSE_SMINMAX_MODE
2562 [(V4SF "TARGET_SSE") (V2DF "TARGET_SSE") 2829 [(V4SF "TARGET_SSE") (V2DF "TARGET_SSE")
2563 (V2DI "TARGET_SSE") (V4SI "TARGET_SSE") (V8HI "TARGET_SSE") 2830 (V2DI "TARGET_SSE4_2") (V4SI "TARGET_SSE") (V8HI "TARGET_SSE")
2564 (V16QI "TARGET_SSE")]) 2831 (V16QI "TARGET_SSE")])
2565 2832
2566 (define_expand "reduc_<code>_scal_<mode>" 2833 (define_expand "reduc_<code>_scal_<mode>"
2567 [(smaxmin:REDUC_SSE_SMINMAX_MODE 2834 [(smaxmin:REDUC_SSE_SMINMAX_MODE
2568 (match_operand:<ssescalarmode> 0 "register_operand") 2835 (match_operand:<ssescalarmode> 0 "register_operand")
2744 (define_insn "<sse>_vmmaskcmp<mode>3" 3011 (define_insn "<sse>_vmmaskcmp<mode>3"
2745 [(set (match_operand:VF_128 0 "register_operand" "=x,x") 3012 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2746 (vec_merge:VF_128 3013 (vec_merge:VF_128
2747 (match_operator:VF_128 3 "sse_comparison_operator" 3014 (match_operator:VF_128 3 "sse_comparison_operator"
2748 [(match_operand:VF_128 1 "register_operand" "0,x") 3015 [(match_operand:VF_128 1 "register_operand" "0,x")
2749 (match_operand:VF_128 2 "vector_operand" "xBm,xm")]) 3016 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm")])
2750 (match_dup 1) 3017 (match_dup 1)
2751 (const_int 1)))] 3018 (const_int 1)))]
2752 "TARGET_SSE" 3019 "TARGET_SSE"
2753 "@ 3020 "@
2754 cmp%D3<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2} 3021 cmp%D3<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
2769 (V32HI "const_0_to_7_operand") (V64QI "const_0_to_7_operand") 3036 (V32HI "const_0_to_7_operand") (V64QI "const_0_to_7_operand")
2770 (V16HI "const_0_to_7_operand") (V32QI "const_0_to_7_operand") 3037 (V16HI "const_0_to_7_operand") (V32QI "const_0_to_7_operand")
2771 (V8HI "const_0_to_7_operand") (V16QI "const_0_to_7_operand")]) 3038 (V8HI "const_0_to_7_operand") (V16QI "const_0_to_7_operand")])
2772 3039
2773 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name><round_saeonly_name>" 3040 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name><round_saeonly_name>"
2774 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3041 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2775 (unspec:<avx512fmaskmode> 3042 (unspec:<avx512fmaskmode>
2776 [(match_operand:V48_AVX512VL 1 "register_operand" "v") 3043 [(match_operand:V48_AVX512VL 1 "register_operand" "v")
2777 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "<round_saeonly_constraint>") 3044 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "<round_saeonly_constraint>")
2778 (match_operand:SI 3 "<cmp_imm_predicate>" "n")] 3045 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2779 UNSPEC_PCMP))] 3046 UNSPEC_PCMP))]
2782 [(set_attr "type" "ssecmp") 3049 [(set_attr "type" "ssecmp")
2783 (set_attr "length_immediate" "1") 3050 (set_attr "length_immediate" "1")
2784 (set_attr "prefix" "evex") 3051 (set_attr "prefix" "evex")
2785 (set_attr "mode" "<sseinsnmode>")]) 3052 (set_attr "mode" "<sseinsnmode>")])
2786 3053
3054 (define_insn "*<avx512>_cmp<mode>3<mask_scalar_merge_name><round_saeonly_name>"
3055 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
3056 (match_operator:<avx512fmaskmode> 3 "ix86_comparison_int_operator"
3057 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
3058 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "<round_saeonly_constraint>")]))]
3059 "TARGET_AVX512F && <round_saeonly_mode512bit_condition>"
3060 "vpcmp<ssemodesuffix>\t{%I3, <round_saeonly_mask_scalar_merge_op4>%2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2<round_saeonly_mask_scalar_merge_op4>, %I3}"
3061 [(set_attr "type" "ssecmp")
3062 (set_attr "length_immediate" "1")
3063 (set_attr "prefix" "evex")
3064 (set_attr "mode" "<sseinsnmode>")])
3065
2787 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name>" 3066 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name>"
2788 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3067 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2789 (unspec:<avx512fmaskmode> 3068 (unspec:<avx512fmaskmode>
2790 [(match_operand:VI12_AVX512VL 1 "register_operand" "v") 3069 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2791 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm") 3070 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2792 (match_operand:SI 3 "<cmp_imm_predicate>" "n")] 3071 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2793 UNSPEC_PCMP))] 3072 UNSPEC_PCMP))]
2796 [(set_attr "type" "ssecmp") 3075 [(set_attr "type" "ssecmp")
2797 (set_attr "length_immediate" "1") 3076 (set_attr "length_immediate" "1")
2798 (set_attr "prefix" "evex") 3077 (set_attr "prefix" "evex")
2799 (set_attr "mode" "<sseinsnmode>")]) 3078 (set_attr "mode" "<sseinsnmode>")])
2800 3079
3080 (define_insn "*<avx512>_cmp<mode>3<mask_scalar_merge_name>"
3081 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
3082 (match_operator:<avx512fmaskmode> 3 "ix86_comparison_int_operator"
3083 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
3084 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]))]
3085 "TARGET_AVX512BW"
3086 "vpcmp<ssemodesuffix>\t{%I3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %I3}"
3087 [(set_attr "type" "ssecmp")
3088 (set_attr "length_immediate" "1")
3089 (set_attr "prefix" "evex")
3090 (set_attr "mode" "<sseinsnmode>")])
3091
2801 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>" 3092 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2802 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3093 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2803 (unspec:<avx512fmaskmode> 3094 (unspec:<avx512fmaskmode>
2804 [(match_operand:VI12_AVX512VL 1 "register_operand" "v") 3095 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2805 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm") 3096 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2806 (match_operand:SI 3 "const_0_to_7_operand" "n")] 3097 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2807 UNSPEC_UNSIGNED_PCMP))] 3098 UNSPEC_UNSIGNED_PCMP))]
2810 [(set_attr "type" "ssecmp") 3101 [(set_attr "type" "ssecmp")
2811 (set_attr "length_immediate" "1") 3102 (set_attr "length_immediate" "1")
2812 (set_attr "prefix" "evex") 3103 (set_attr "prefix" "evex")
2813 (set_attr "mode" "<sseinsnmode>")]) 3104 (set_attr "mode" "<sseinsnmode>")])
2814 3105
3106 (define_insn "*<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
3107 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
3108 (match_operator:<avx512fmaskmode> 3 "ix86_comparison_uns_operator"
3109 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
3110 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]))]
3111 "TARGET_AVX512BW"
3112 "vpcmpu<ssemodesuffix>\t{%I3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %I3}"
3113 [(set_attr "type" "ssecmp")
3114 (set_attr "length_immediate" "1")
3115 (set_attr "prefix" "evex")
3116 (set_attr "mode" "<sseinsnmode>")])
3117
2815 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>" 3118 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2816 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3119 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2817 (unspec:<avx512fmaskmode> 3120 (unspec:<avx512fmaskmode>
2818 [(match_operand:VI48_AVX512VL 1 "register_operand" "v") 3121 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
2819 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm") 3122 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")
2820 (match_operand:SI 3 "const_0_to_7_operand" "n")] 3123 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2821 UNSPEC_UNSIGNED_PCMP))] 3124 UNSPEC_UNSIGNED_PCMP))]
2824 [(set_attr "type" "ssecmp") 3127 [(set_attr "type" "ssecmp")
2825 (set_attr "length_immediate" "1") 3128 (set_attr "length_immediate" "1")
2826 (set_attr "prefix" "evex") 3129 (set_attr "prefix" "evex")
2827 (set_attr "mode" "<sseinsnmode>")]) 3130 (set_attr "mode" "<sseinsnmode>")])
2828 3131
3132 (define_insn "*<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
3133 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
3134 (match_operator:<avx512fmaskmode> 3 "ix86_comparison_uns_operator"
3135 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
3136 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]))]
3137 "TARGET_AVX512F"
3138 "vpcmpu<ssemodesuffix>\t{%I3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %I3}"
3139 [(set_attr "type" "ssecmp")
3140 (set_attr "length_immediate" "1")
3141 (set_attr "prefix" "evex")
3142 (set_attr "mode" "<sseinsnmode>")])
3143
2829 (define_insn "avx512f_vmcmp<mode>3<round_saeonly_name>" 3144 (define_insn "avx512f_vmcmp<mode>3<round_saeonly_name>"
2830 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3145 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2831 (and:<avx512fmaskmode> 3146 (and:<avx512fmaskmode>
2832 (unspec:<avx512fmaskmode> 3147 (unspec:<avx512fmaskmode>
2833 [(match_operand:VF_128 1 "register_operand" "v") 3148 [(match_operand:VF_128 1 "register_operand" "v")
2834 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>") 3149 (match_operand:VF_128 2 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
2835 (match_operand:SI 3 "const_0_to_31_operand" "n")] 3150 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2836 UNSPEC_PCMP) 3151 UNSPEC_PCMP)
2837 (const_int 1)))] 3152 (const_int 1)))]
2838 "TARGET_AVX512F" 3153 "TARGET_AVX512F"
2839 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %<iptr>2<round_saeonly_op4>, %3}" 3154 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %<iptr>2<round_saeonly_op4>, %3}"
2841 (set_attr "length_immediate" "1") 3156 (set_attr "length_immediate" "1")
2842 (set_attr "prefix" "evex") 3157 (set_attr "prefix" "evex")
2843 (set_attr "mode" "<ssescalarmode>")]) 3158 (set_attr "mode" "<ssescalarmode>")])
2844 3159
2845 (define_insn "avx512f_vmcmp<mode>3_mask<round_saeonly_name>" 3160 (define_insn "avx512f_vmcmp<mode>3_mask<round_saeonly_name>"
2846 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3161 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2847 (and:<avx512fmaskmode> 3162 (and:<avx512fmaskmode>
2848 (unspec:<avx512fmaskmode> 3163 (unspec:<avx512fmaskmode>
2849 [(match_operand:VF_128 1 "register_operand" "v") 3164 [(match_operand:VF_128 1 "register_operand" "v")
2850 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>") 3165 (match_operand:VF_128 2 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
2851 (match_operand:SI 3 "const_0_to_31_operand" "n")] 3166 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2852 UNSPEC_PCMP) 3167 UNSPEC_PCMP)
2853 (and:<avx512fmaskmode> 3168 (and:<avx512fmaskmode>
2854 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk") 3169 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")
2855 (const_int 1))))] 3170 (const_int 1))))]
2859 (set_attr "length_immediate" "1") 3174 (set_attr "length_immediate" "1")
2860 (set_attr "prefix" "evex") 3175 (set_attr "prefix" "evex")
2861 (set_attr "mode" "<ssescalarmode>")]) 3176 (set_attr "mode" "<ssescalarmode>")])
2862 3177
2863 (define_insn "avx512f_maskcmp<mode>3" 3178 (define_insn "avx512f_maskcmp<mode>3"
2864 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 3179 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
2865 (match_operator:<avx512fmaskmode> 3 "sse_comparison_operator" 3180 (match_operator:<avx512fmaskmode> 3 "sse_comparison_operator"
2866 [(match_operand:VF 1 "register_operand" "v") 3181 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
2867 (match_operand:VF 2 "nonimmediate_operand" "vm")]))] 3182 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "vm")]))]
2868 "TARGET_AVX512F" 3183 "TARGET_AVX512F"
2869 "vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}" 3184 "vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2870 [(set_attr "type" "ssecmp") 3185 [(set_attr "type" "ssecmp")
2871 (set_attr "length_immediate" "1") 3186 (set_attr "length_immediate" "1")
2872 (set_attr "prefix" "evex") 3187 (set_attr "prefix" "evex")
3113 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand") 3428 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand")
3114 (match_operand:VI12_AVX512VL 2 "nonimm_or_0_operand") 3429 (match_operand:VI12_AVX512VL 2 "nonimm_or_0_operand")
3115 (match_operand:<avx512fmaskmode> 3 "register_operand")))] 3430 (match_operand:<avx512fmaskmode> 3 "register_operand")))]
3116 "TARGET_AVX512BW") 3431 "TARGET_AVX512BW")
3117 3432
3433 ;; As vcondv4div4df and vcondv8siv8sf are enabled already with TARGET_AVX,
3434 ;; and their condition can be folded late into a constant, we need to
3435 ;; support vcond_mask_v4div4di and vcond_mask_v8siv8si for TARGET_AVX.
3436 (define_mode_iterator VI_256_AVX2 [(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
3437 V8SI V4DI])
3438
3118 (define_expand "vcond_mask_<mode><sseintvecmodelower>" 3439 (define_expand "vcond_mask_<mode><sseintvecmodelower>"
3119 [(set (match_operand:VI_256 0 "register_operand") 3440 [(set (match_operand:VI_256_AVX2 0 "register_operand")
3120 (vec_merge:VI_256 3441 (vec_merge:VI_256_AVX2
3121 (match_operand:VI_256 1 "nonimmediate_operand") 3442 (match_operand:VI_256_AVX2 1 "nonimmediate_operand")
3122 (match_operand:VI_256 2 "nonimm_or_0_operand") 3443 (match_operand:VI_256_AVX2 2 "nonimm_or_0_operand")
3123 (match_operand:<sseintvecmode> 3 "register_operand")))] 3444 (match_operand:<sseintvecmode> 3 "register_operand")))]
3124 "TARGET_AVX2" 3445 "TARGET_AVX"
3125 { 3446 {
3126 ix86_expand_sse_movcc (operands[0], operands[3], 3447 ix86_expand_sse_movcc (operands[0], operands[3],
3127 operands[1], operands[2]); 3448 operands[1], operands[2]);
3128 DONE; 3449 DONE;
3129 }) 3450 })
3192 (not:VF_128_256 3513 (not:VF_128_256
3193 (match_operand:VF_128_256 1 "register_operand" "0,x,v,v")) 3514 (match_operand:VF_128_256 1 "register_operand" "0,x,v,v"))
3194 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))] 3515 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
3195 "TARGET_SSE && <mask_avx512vl_condition>" 3516 "TARGET_SSE && <mask_avx512vl_condition>"
3196 { 3517 {
3197 static char buf[128]; 3518 char buf[128];
3198 const char *ops; 3519 const char *ops;
3199 const char *suffix; 3520 const char *suffix;
3200 3521
3201 switch (which_alternative) 3522 switch (which_alternative)
3202 { 3523 {
3227 default: 3548 default:
3228 suffix = "<ssemodesuffix>"; 3549 suffix = "<ssemodesuffix>";
3229 } 3550 }
3230 3551
3231 snprintf (buf, sizeof (buf), ops, suffix); 3552 snprintf (buf, sizeof (buf), ops, suffix);
3232 return buf; 3553 output_asm_insn (buf, operands);
3554 return "";
3233 } 3555 }
3234 [(set_attr "isa" "noavx,avx,avx512dq,avx512f") 3556 [(set_attr "isa" "noavx,avx,avx512dq,avx512f")
3235 (set_attr "type" "sselog") 3557 (set_attr "type" "sselog")
3236 (set_attr "prefix" "orig,maybe_vex,evex,evex") 3558 (set_attr "prefix" "orig,maybe_vex,evex,evex")
3237 (set (attr "mode") 3559 (set (attr "mode")
3239 (and (eq_attr "alternative" "1") 3561 (and (eq_attr "alternative" "1")
3240 (match_test "!TARGET_AVX512DQ"))) 3562 (match_test "!TARGET_AVX512DQ")))
3241 (const_string "<sseintvecmode2>") 3563 (const_string "<sseintvecmode2>")
3242 (eq_attr "alternative" "3") 3564 (eq_attr "alternative" "3")
3243 (const_string "<sseintvecmode2>") 3565 (const_string "<sseintvecmode2>")
3244 (and (match_test "<MODE_SIZE> == 16")
3245 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3246 (const_string "<ssePSmode>")
3247 (match_test "TARGET_AVX") 3566 (match_test "TARGET_AVX")
3248 (const_string "<MODE>") 3567 (const_string "<MODE>")
3249 (match_test "optimize_function_for_size_p (cfun)") 3568 (match_test "optimize_function_for_size_p (cfun)")
3250 (const_string "V4SF") 3569 (const_string "V4SF")
3251 ] 3570 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3252 (const_string "<MODE>")))]) 3571 (const_string "V4SF")
3253 3572 ]
3573 (const_string "<MODE>")))])
3254 3574
3255 (define_insn "<sse>_andnot<mode>3<mask_name>" 3575 (define_insn "<sse>_andnot<mode>3<mask_name>"
3256 [(set (match_operand:VF_512 0 "register_operand" "=v") 3576 [(set (match_operand:VF_512 0 "register_operand" "=v")
3257 (and:VF_512 3577 (and:VF_512
3258 (not:VF_512 3578 (not:VF_512
3259 (match_operand:VF_512 1 "register_operand" "v")) 3579 (match_operand:VF_512 1 "register_operand" "v"))
3260 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))] 3580 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
3261 "TARGET_AVX512F" 3581 "TARGET_AVX512F"
3262 { 3582 {
3263 static char buf[128]; 3583 char buf[128];
3264 const char *ops; 3584 const char *ops;
3265 const char *suffix; 3585 const char *suffix;
3266 3586
3267 suffix = "<ssemodesuffix>"; 3587 suffix = "<ssemodesuffix>";
3268 ops = ""; 3588 ops = "";
3275 } 3595 }
3276 3596
3277 snprintf (buf, sizeof (buf), 3597 snprintf (buf, sizeof (buf),
3278 "v%sandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}", 3598 "v%sandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
3279 ops, suffix); 3599 ops, suffix);
3280 return buf; 3600 output_asm_insn (buf, operands);
3601 return "";
3281 } 3602 }
3282 [(set_attr "type" "sselog") 3603 [(set_attr "type" "sselog")
3283 (set_attr "prefix" "evex") 3604 (set_attr "prefix" "evex")
3284 (set (attr "mode") 3605 (set (attr "mode")
3285 (if_then_else (match_test "TARGET_AVX512DQ") 3606 (if_then_else (match_test "TARGET_AVX512DQ")
3308 (match_operand:VF_128_256 1 "vector_operand" "%0,x,v,v") 3629 (match_operand:VF_128_256 1 "vector_operand" "%0,x,v,v")
3309 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))] 3630 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
3310 "TARGET_SSE && <mask_avx512vl_condition> 3631 "TARGET_SSE && <mask_avx512vl_condition>
3311 && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 3632 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3312 { 3633 {
3313 static char buf[128]; 3634 char buf[128];
3314 const char *ops; 3635 const char *ops;
3315 const char *suffix; 3636 const char *suffix;
3316 3637
3317 switch (which_alternative) 3638 switch (which_alternative)
3318 { 3639 {
3343 default: 3664 default:
3344 suffix = "<ssemodesuffix>"; 3665 suffix = "<ssemodesuffix>";
3345 } 3666 }
3346 3667
3347 snprintf (buf, sizeof (buf), ops, suffix); 3668 snprintf (buf, sizeof (buf), ops, suffix);
3348 return buf; 3669 output_asm_insn (buf, operands);
3670 return "";
3349 } 3671 }
3350 [(set_attr "isa" "noavx,avx,avx512dq,avx512f") 3672 [(set_attr "isa" "noavx,avx,avx512dq,avx512f")
3351 (set_attr "type" "sselog") 3673 (set_attr "type" "sselog")
3352 (set_attr "prefix" "orig,maybe_evex,evex,evex") 3674 (set_attr "prefix" "orig,maybe_evex,evex,evex")
3353 (set (attr "mode") 3675 (set (attr "mode")
3355 (and (eq_attr "alternative" "1") 3677 (and (eq_attr "alternative" "1")
3356 (match_test "!TARGET_AVX512DQ"))) 3678 (match_test "!TARGET_AVX512DQ")))
3357 (const_string "<sseintvecmode2>") 3679 (const_string "<sseintvecmode2>")
3358 (eq_attr "alternative" "3") 3680 (eq_attr "alternative" "3")
3359 (const_string "<sseintvecmode2>") 3681 (const_string "<sseintvecmode2>")
3360 (and (match_test "<MODE_SIZE> == 16")
3361 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3362 (const_string "<ssePSmode>")
3363 (match_test "TARGET_AVX") 3682 (match_test "TARGET_AVX")
3364 (const_string "<MODE>") 3683 (const_string "<MODE>")
3365 (match_test "optimize_function_for_size_p (cfun)") 3684 (match_test "optimize_function_for_size_p (cfun)")
3366 (const_string "V4SF") 3685 (const_string "V4SF")
3367 ] 3686 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3368 (const_string "<MODE>")))]) 3687 (const_string "V4SF")
3688 ]
3689 (const_string "<MODE>")))])
3369 3690
3370 (define_insn "*<code><mode>3<mask_name>" 3691 (define_insn "*<code><mode>3<mask_name>"
3371 [(set (match_operand:VF_512 0 "register_operand" "=v") 3692 [(set (match_operand:VF_512 0 "register_operand" "=v")
3372 (any_logic:VF_512 3693 (any_logic:VF_512
3373 (match_operand:VF_512 1 "nonimmediate_operand" "%v") 3694 (match_operand:VF_512 1 "nonimmediate_operand" "%v")
3374 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))] 3695 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
3375 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 3696 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3376 { 3697 {
3377 static char buf[128]; 3698 char buf[128];
3378 const char *ops; 3699 const char *ops;
3379 const char *suffix; 3700 const char *suffix;
3380 3701
3381 suffix = "<ssemodesuffix>"; 3702 suffix = "<ssemodesuffix>";
3382 ops = ""; 3703 ops = "";
3389 } 3710 }
3390 3711
3391 snprintf (buf, sizeof (buf), 3712 snprintf (buf, sizeof (buf),
3392 "v%s<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}", 3713 "v%s<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
3393 ops, suffix); 3714 ops, suffix);
3394 return buf; 3715 output_asm_insn (buf, operands);
3716 return "";
3395 } 3717 }
3396 [(set_attr "type" "sselog") 3718 [(set_attr "type" "sselog")
3397 (set_attr "prefix" "evex") 3719 (set_attr "prefix" "evex")
3398 (set (attr "mode") 3720 (set (attr "mode")
3399 (if_then_else (match_test "TARGET_AVX512DQ") 3721 (if_then_else (match_test "TARGET_AVX512DQ")
3416 3738
3417 operands[4] = gen_reg_rtx (<MODE>mode); 3739 operands[4] = gen_reg_rtx (<MODE>mode);
3418 operands[5] = gen_reg_rtx (<MODE>mode); 3740 operands[5] = gen_reg_rtx (<MODE>mode);
3419 }) 3741 })
3420 3742
3743 (define_expand "xorsign<mode>3"
3744 [(set (match_dup 4)
3745 (and:VF (match_dup 3)
3746 (match_operand:VF 2 "vector_operand")))
3747 (set (match_operand:VF 0 "register_operand")
3748 (xor:VF (match_dup 4)
3749 (match_operand:VF 1 "vector_operand")))]
3750 "TARGET_SSE"
3751 {
3752 operands[3] = ix86_build_signbit_mask (<MODE>mode, 1, 0);
3753
3754 operands[4] = gen_reg_rtx (<MODE>mode);
3755 })
3756
3757 (define_expand "signbit<mode>2"
3758 [(set (match_operand:<sseintvecmode> 0 "register_operand")
3759 (lshiftrt:<sseintvecmode>
3760 (subreg:<sseintvecmode>
3761 (match_operand:VF1_AVX2 1 "register_operand") 0)
3762 (match_dup 2)))]
3763 "TARGET_SSE2"
3764 "operands[2] = GEN_INT (GET_MODE_UNIT_BITSIZE (<MODE>mode)-1);")
3765
3421 ;; Also define scalar versions. These are used for abs, neg, and 3766 ;; Also define scalar versions. These are used for abs, neg, and
3422 ;; conditional move. Using subregs into vector modes causes register 3767 ;; conditional move. Using subregs into vector modes causes register
3423 ;; allocation lossage. These patterns do not allow memory operands 3768 ;; allocation lossage. These patterns do not allow memory operands
3424 ;; because the native instructions read the full 128-bits. 3769 ;; because the native instructions read the full 128-bits.
3425 3770
3429 (not:MODEF 3774 (not:MODEF
3430 (match_operand:MODEF 1 "register_operand" "0,x,v,v")) 3775 (match_operand:MODEF 1 "register_operand" "0,x,v,v"))
3431 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))] 3776 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))]
3432 "SSE_FLOAT_MODE_P (<MODE>mode)" 3777 "SSE_FLOAT_MODE_P (<MODE>mode)"
3433 { 3778 {
3434 static char buf[128]; 3779 char buf[128];
3435 const char *ops; 3780 const char *ops;
3436 const char *suffix 3781 const char *suffix
3437 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>"; 3782 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
3438 3783
3439 switch (which_alternative) 3784 switch (which_alternative)
3465 default: 3810 default:
3466 gcc_unreachable (); 3811 gcc_unreachable ();
3467 } 3812 }
3468 3813
3469 snprintf (buf, sizeof (buf), ops, suffix); 3814 snprintf (buf, sizeof (buf), ops, suffix);
3470 return buf; 3815 output_asm_insn (buf, operands);
3816 return "";
3471 } 3817 }
3472 [(set_attr "isa" "noavx,avx,avx512vl,avx512f") 3818 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3473 (set_attr "type" "sselog") 3819 (set_attr "type" "sselog")
3474 (set_attr "prefix" "orig,vex,evex,evex") 3820 (set_attr "prefix" "orig,vex,evex,evex")
3475 (set (attr "mode") 3821 (set (attr "mode")
3479 (const_string "TI")) 3825 (const_string "TI"))
3480 (eq_attr "alternative" "3") 3826 (eq_attr "alternative" "3")
3481 (if_then_else (match_test "TARGET_AVX512DQ") 3827 (if_then_else (match_test "TARGET_AVX512DQ")
3482 (const_string "<avx512fvecmode>") 3828 (const_string "<avx512fvecmode>")
3483 (const_string "XI")) 3829 (const_string "XI"))
3484 (and (match_test "<MODE_SIZE> == 16")
3485 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3486 (const_string "V4SF")
3487 (match_test "TARGET_AVX") 3830 (match_test "TARGET_AVX")
3488 (const_string "<ssevecmode>") 3831 (const_string "<ssevecmode>")
3489 (match_test "optimize_function_for_size_p (cfun)") 3832 (match_test "optimize_function_for_size_p (cfun)")
3490 (const_string "V4SF") 3833 (const_string "V4SF")
3491 ] 3834 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3492 (const_string "<ssevecmode>")))]) 3835 (const_string "V4SF")
3836 ]
3837 (const_string "<ssevecmode>")))])
3493 3838
3494 (define_insn "*andnottf3" 3839 (define_insn "*andnottf3"
3495 [(set (match_operand:TF 0 "register_operand" "=x,x,v,v") 3840 [(set (match_operand:TF 0 "register_operand" "=x,x,v,v")
3496 (and:TF 3841 (and:TF
3497 (not:TF (match_operand:TF 1 "register_operand" "0,x,v,v")) 3842 (not:TF (match_operand:TF 1 "register_operand" "0,x,v,v"))
3498 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))] 3843 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))]
3499 "TARGET_SSE" 3844 "TARGET_SSE"
3500 { 3845 {
3501 static char buf[128]; 3846 char buf[128];
3502 const char *ops; 3847 const char *ops;
3503 const char *tmp 3848 const char *tmp
3504 = (which_alternative >= 2 ? "pandnq" 3849 = (which_alternative >= 2 ? "pandnq"
3505 : get_attr_mode (insn) == MODE_V4SF ? "andnps" : "pandn"); 3850 : get_attr_mode (insn) == MODE_V4SF ? "andnps" : "pandn");
3506 3851
3519 default: 3864 default:
3520 gcc_unreachable (); 3865 gcc_unreachable ();
3521 } 3866 }
3522 3867
3523 snprintf (buf, sizeof (buf), ops, tmp); 3868 snprintf (buf, sizeof (buf), ops, tmp);
3524 return buf; 3869 output_asm_insn (buf, operands);
3870 return "";
3525 } 3871 }
3526 [(set_attr "isa" "noavx,avx,avx512vl,avx512f") 3872 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3527 (set_attr "type" "sselog") 3873 (set_attr "type" "sselog")
3528 (set (attr "prefix_data16") 3874 (set (attr "prefix_data16")
3529 (if_then_else 3875 (if_then_else
3535 (set (attr "mode") 3881 (set (attr "mode")
3536 (cond [(eq_attr "alternative" "2") 3882 (cond [(eq_attr "alternative" "2")
3537 (const_string "TI") 3883 (const_string "TI")
3538 (eq_attr "alternative" "3") 3884 (eq_attr "alternative" "3")
3539 (const_string "XI") 3885 (const_string "XI")
3540 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3541 (const_string "V4SF")
3542 (match_test "TARGET_AVX") 3886 (match_test "TARGET_AVX")
3543 (const_string "TI") 3887 (const_string "TI")
3544 (ior (not (match_test "TARGET_SSE2")) 3888 (ior (not (match_test "TARGET_SSE2"))
3545 (match_test "optimize_function_for_size_p (cfun)")) 3889 (match_test "optimize_function_for_size_p (cfun)"))
3546 (const_string "V4SF") 3890 (const_string "V4SF")
3547 ] 3891 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3548 (const_string "TI")))]) 3892 (const_string "V4SF")
3893 ]
3894 (const_string "TI")))])
3549 3895
3550 (define_insn "*<code><mode>3" 3896 (define_insn "*<code><mode>3"
3551 [(set (match_operand:MODEF 0 "register_operand" "=x,x,v,v") 3897 [(set (match_operand:MODEF 0 "register_operand" "=x,x,v,v")
3552 (any_logic:MODEF 3898 (any_logic:MODEF
3553 (match_operand:MODEF 1 "register_operand" "%0,x,v,v") 3899 (match_operand:MODEF 1 "register_operand" "%0,x,v,v")
3554 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))] 3900 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))]
3555 "SSE_FLOAT_MODE_P (<MODE>mode)" 3901 "SSE_FLOAT_MODE_P (<MODE>mode)"
3556 { 3902 {
3557 static char buf[128]; 3903 char buf[128];
3558 const char *ops; 3904 const char *ops;
3559 const char *suffix 3905 const char *suffix
3560 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>"; 3906 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
3561 3907
3562 switch (which_alternative) 3908 switch (which_alternative)
3587 default: 3933 default:
3588 gcc_unreachable (); 3934 gcc_unreachable ();
3589 } 3935 }
3590 3936
3591 snprintf (buf, sizeof (buf), ops, suffix); 3937 snprintf (buf, sizeof (buf), ops, suffix);
3592 return buf; 3938 output_asm_insn (buf, operands);
3939 return "";
3593 } 3940 }
3594 [(set_attr "isa" "noavx,avx,avx512vl,avx512f") 3941 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3595 (set_attr "type" "sselog") 3942 (set_attr "type" "sselog")
3596 (set_attr "prefix" "orig,vex,evex,evex") 3943 (set_attr "prefix" "orig,vex,evex,evex")
3597 (set (attr "mode") 3944 (set (attr "mode")
3601 (const_string "TI")) 3948 (const_string "TI"))
3602 (eq_attr "alternative" "3") 3949 (eq_attr "alternative" "3")
3603 (if_then_else (match_test "TARGET_AVX512DQ") 3950 (if_then_else (match_test "TARGET_AVX512DQ")
3604 (const_string "<avx512fvecmode>") 3951 (const_string "<avx512fvecmode>")
3605 (const_string "XI")) 3952 (const_string "XI"))
3606 (and (match_test "<MODE_SIZE> == 16")
3607 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3608 (const_string "V4SF")
3609 (match_test "TARGET_AVX") 3953 (match_test "TARGET_AVX")
3610 (const_string "<ssevecmode>") 3954 (const_string "<ssevecmode>")
3611 (match_test "optimize_function_for_size_p (cfun)") 3955 (match_test "optimize_function_for_size_p (cfun)")
3612 (const_string "V4SF") 3956 (const_string "V4SF")
3613 ] 3957 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3614 (const_string "<ssevecmode>")))]) 3958 (const_string "V4SF")
3959 ]
3960 (const_string "<ssevecmode>")))])
3615 3961
3616 (define_expand "<code>tf3" 3962 (define_expand "<code>tf3"
3617 [(set (match_operand:TF 0 "register_operand") 3963 [(set (match_operand:TF 0 "register_operand")
3618 (any_logic:TF 3964 (any_logic:TF
3619 (match_operand:TF 1 "vector_operand") 3965 (match_operand:TF 1 "vector_operand")
3626 (any_logic:TF 3972 (any_logic:TF
3627 (match_operand:TF 1 "vector_operand" "%0,x,v,v") 3973 (match_operand:TF 1 "vector_operand" "%0,x,v,v")
3628 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))] 3974 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))]
3629 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 3975 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3630 { 3976 {
3631 static char buf[128]; 3977 char buf[128];
3632 const char *ops; 3978 const char *ops;
3633 const char *tmp 3979 const char *tmp
3634 = (which_alternative >= 2 ? "p<logic>q" 3980 = (which_alternative >= 2 ? "p<logic>q"
3635 : get_attr_mode (insn) == MODE_V4SF ? "<logic>ps" : "p<logic>"); 3981 : get_attr_mode (insn) == MODE_V4SF ? "<logic>ps" : "p<logic>");
3636 3982
3649 default: 3995 default:
3650 gcc_unreachable (); 3996 gcc_unreachable ();
3651 } 3997 }
3652 3998
3653 snprintf (buf, sizeof (buf), ops, tmp); 3999 snprintf (buf, sizeof (buf), ops, tmp);
3654 return buf; 4000 output_asm_insn (buf, operands);
4001 return "";
3655 } 4002 }
3656 [(set_attr "isa" "noavx,avx,avx512vl,avx512f") 4003 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3657 (set_attr "type" "sselog") 4004 (set_attr "type" "sselog")
3658 (set (attr "prefix_data16") 4005 (set (attr "prefix_data16")
3659 (if_then_else 4006 (if_then_else
3665 (set (attr "mode") 4012 (set (attr "mode")
3666 (cond [(eq_attr "alternative" "2") 4013 (cond [(eq_attr "alternative" "2")
3667 (const_string "TI") 4014 (const_string "TI")
3668 (eq_attr "alternative" "3") 4015 (eq_attr "alternative" "3")
3669 (const_string "QI") 4016 (const_string "QI")
3670 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3671 (const_string "V4SF")
3672 (match_test "TARGET_AVX") 4017 (match_test "TARGET_AVX")
3673 (const_string "TI") 4018 (const_string "TI")
3674 (ior (not (match_test "TARGET_SSE2")) 4019 (ior (not (match_test "TARGET_SSE2"))
3675 (match_test "optimize_function_for_size_p (cfun)")) 4020 (match_test "optimize_function_for_size_p (cfun)"))
3676 (const_string "V4SF") 4021 (const_string "V4SF")
3677 ] 4022 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3678 (const_string "TI")))]) 4023 (const_string "V4SF")
4024 ]
4025 (const_string "TI")))])
3679 4026
3680 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 4027 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3681 ;; 4028 ;;
3682 ;; FMA floating point multiply/accumulate instructions. These include 4029 ;; FMA floating point multiply/accumulate instructions. These include
3683 ;; scalar versions of the instructions as well as vector versions. 4030 ;; scalar versions of the instructions as well as vector versions.
3825 vfmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}" 4172 vfmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3826 [(set_attr "type" "ssemuladd") 4173 [(set_attr "type" "ssemuladd")
3827 (set_attr "mode" "<MODE>")]) 4174 (set_attr "mode" "<MODE>")])
3828 4175
3829 (define_insn "*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_1" 4176 (define_insn "*<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name>_bcst_1"
3830 [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v") 4177 [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
3831 (fma:VF_AVX512 4178 (fma:VF_AVX512
3832 (match_operand:VF_AVX512 1 "register_operand" "0,v") 4179 (match_operand:VF_AVX512 1 "register_operand" "%0")
3833 (match_operand:VF_AVX512 2 "register_operand" "v,0") 4180 (match_operand:VF_AVX512 2 "register_operand" "v")
3834 (vec_duplicate:VF_AVX512 4181 (vec_duplicate:VF_AVX512
3835 (match_operand:<ssescalarmode> 3 "memory_operand" "m,m"))))] 4182 (match_operand:<ssescalarmode> 3 "memory_operand" "m"))))]
3836 "TARGET_AVX512F && <sd_mask_mode512bit_condition>" 4183 "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
3837 "vfmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}" 4184 "vfmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
3838 [(set_attr "type" "ssemuladd") 4185 [(set_attr "type" "ssemuladd")
3839 (set_attr "mode" "<MODE>")]) 4186 (set_attr "mode" "<MODE>")])
3840 4187
3869 (define_insn "<avx512>_fmadd_<mode>_mask<round_name>" 4216 (define_insn "<avx512>_fmadd_<mode>_mask<round_name>"
3870 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4217 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3871 (vec_merge:VF_AVX512VL 4218 (vec_merge:VF_AVX512VL
3872 (fma:VF_AVX512VL 4219 (fma:VF_AVX512VL
3873 (match_operand:VF_AVX512VL 1 "register_operand" "0,0") 4220 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3874 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4221 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
3875 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")) 4222 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>"))
3876 (match_dup 1) 4223 (match_dup 1)
3877 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4224 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3878 "TARGET_AVX512F && <round_mode512bit_condition>" 4225 "TARGET_AVX512F && <round_mode512bit_condition>"
3879 "@ 4226 "@
3880 vfmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>} 4227 vfmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3884 4231
3885 (define_insn "<avx512>_fmadd_<mode>_mask3<round_name>" 4232 (define_insn "<avx512>_fmadd_<mode>_mask3<round_name>"
3886 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4233 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3887 (vec_merge:VF_AVX512VL 4234 (vec_merge:VF_AVX512VL
3888 (fma:VF_AVX512VL 4235 (fma:VF_AVX512VL
3889 (match_operand:VF_AVX512VL 1 "register_operand" "v") 4236 (match_operand:VF_AVX512VL 1 "<round_nimm_predicate>" "%v")
3890 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4237 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
3891 (match_operand:VF_AVX512VL 3 "register_operand" "0")) 4238 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
3892 (match_dup 3) 4239 (match_dup 3)
3893 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4240 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3894 "TARGET_AVX512F" 4241 "TARGET_AVX512F"
3895 "vfmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}" 4242 "vfmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3942 vfmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}" 4289 vfmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3943 [(set_attr "type" "ssemuladd") 4290 [(set_attr "type" "ssemuladd")
3944 (set_attr "mode" "<MODE>")]) 4291 (set_attr "mode" "<MODE>")])
3945 4292
3946 (define_insn "*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_1" 4293 (define_insn "*<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name>_bcst_1"
3947 [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v") 4294 [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
3948 (fma:VF_AVX512 4295 (fma:VF_AVX512
3949 (match_operand:VF_AVX512 1 "register_operand" "0,v") 4296 (match_operand:VF_AVX512 1 "register_operand" "%0")
3950 (match_operand:VF_AVX512 2 "register_operand" "v,0") 4297 (match_operand:VF_AVX512 2 "register_operand" "v")
3951 (neg:VF_AVX512 4298 (neg:VF_AVX512
3952 (vec_duplicate:VF_AVX512 4299 (vec_duplicate:VF_AVX512
3953 (match_operand:<ssescalarmode> 3 "memory_operand" "m,m")))))] 4300 (match_operand:<ssescalarmode> 3 "memory_operand" "m")))))]
3954 "TARGET_AVX512F && <sd_mask_mode512bit_condition>" 4301 "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
3955 "vfmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}" 4302 "vfmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
3956 [(set_attr "type" "ssemuladd") 4303 [(set_attr "type" "ssemuladd")
3957 (set_attr "mode" "<MODE>")]) 4304 (set_attr "mode" "<MODE>")])
3958 4305
3989 (define_insn "<avx512>_fmsub_<mode>_mask<round_name>" 4336 (define_insn "<avx512>_fmsub_<mode>_mask<round_name>"
3990 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4337 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3991 (vec_merge:VF_AVX512VL 4338 (vec_merge:VF_AVX512VL
3992 (fma:VF_AVX512VL 4339 (fma:VF_AVX512VL
3993 (match_operand:VF_AVX512VL 1 "register_operand" "0,0") 4340 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3994 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4341 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
3995 (neg:VF_AVX512VL 4342 (neg:VF_AVX512VL
3996 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))) 4343 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>")))
3997 (match_dup 1) 4344 (match_dup 1)
3998 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4345 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3999 "TARGET_AVX512F" 4346 "TARGET_AVX512F"
4000 "@ 4347 "@
4001 vfmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>} 4348 vfmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
4005 4352
4006 (define_insn "<avx512>_fmsub_<mode>_mask3<round_name>" 4353 (define_insn "<avx512>_fmsub_<mode>_mask3<round_name>"
4007 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4354 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4008 (vec_merge:VF_AVX512VL 4355 (vec_merge:VF_AVX512VL
4009 (fma:VF_AVX512VL 4356 (fma:VF_AVX512VL
4010 (match_operand:VF_AVX512VL 1 "register_operand" "v") 4357 (match_operand:VF_AVX512VL 1 "<round_nimm_predicate>" "%v")
4011 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4358 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
4012 (neg:VF_AVX512VL 4359 (neg:VF_AVX512VL
4013 (match_operand:VF_AVX512VL 3 "register_operand" "0"))) 4360 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
4014 (match_dup 3) 4361 (match_dup 3)
4015 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4362 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4016 "TARGET_AVX512F && <round_mode512bit_condition>" 4363 "TARGET_AVX512F && <round_mode512bit_condition>"
4064 vfnmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}" 4411 vfnmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
4065 [(set_attr "type" "ssemuladd") 4412 [(set_attr "type" "ssemuladd")
4066 (set_attr "mode" "<MODE>")]) 4413 (set_attr "mode" "<MODE>")])
4067 4414
4068 (define_insn "*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_1" 4415 (define_insn "*<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name>_bcst_1"
4069 [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v") 4416 [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
4070 (fma:VF_AVX512 4417 (fma:VF_AVX512
4071 (neg:VF_AVX512 4418 (neg:VF_AVX512
4072 (match_operand:VF_AVX512 1 "register_operand" "0,v")) 4419 (match_operand:VF_AVX512 1 "register_operand" "%0"))
4073 (match_operand:VF_AVX512 2 "register_operand" "v,0") 4420 (match_operand:VF_AVX512 2 "register_operand" "v")
4074 (vec_duplicate:VF_AVX512 4421 (vec_duplicate:VF_AVX512
4075 (match_operand:<ssescalarmode> 3 "memory_operand" "m,m"))))] 4422 (match_operand:<ssescalarmode> 3 "memory_operand" "m"))))]
4076 "TARGET_AVX512F && <sd_mask_mode512bit_condition>" 4423 "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
4077 "vfnmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}" 4424 "vfnmadd213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
4078 [(set_attr "type" "ssemuladd") 4425 [(set_attr "type" "ssemuladd")
4079 (set_attr "mode" "<MODE>")]) 4426 (set_attr "mode" "<MODE>")])
4080 4427
4112 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4459 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4113 (vec_merge:VF_AVX512VL 4460 (vec_merge:VF_AVX512VL
4114 (fma:VF_AVX512VL 4461 (fma:VF_AVX512VL
4115 (neg:VF_AVX512VL 4462 (neg:VF_AVX512VL
4116 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")) 4463 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
4117 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4464 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
4118 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")) 4465 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>"))
4119 (match_dup 1) 4466 (match_dup 1)
4120 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4467 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4121 "TARGET_AVX512F && <round_mode512bit_condition>" 4468 "TARGET_AVX512F && <round_mode512bit_condition>"
4122 "@ 4469 "@
4123 vfnmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>} 4470 vfnmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
4128 (define_insn "<avx512>_fnmadd_<mode>_mask3<round_name>" 4475 (define_insn "<avx512>_fnmadd_<mode>_mask3<round_name>"
4129 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4476 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4130 (vec_merge:VF_AVX512VL 4477 (vec_merge:VF_AVX512VL
4131 (fma:VF_AVX512VL 4478 (fma:VF_AVX512VL
4132 (neg:VF_AVX512VL 4479 (neg:VF_AVX512VL
4133 (match_operand:VF_AVX512VL 1 "register_operand" "v")) 4480 (match_operand:VF_AVX512VL 1 "<round_nimm_predicate>" "%v"))
4134 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4481 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
4135 (match_operand:VF_AVX512VL 3 "register_operand" "0")) 4482 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
4136 (match_dup 3) 4483 (match_dup 3)
4137 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4484 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4138 "TARGET_AVX512F && <round_mode512bit_condition>" 4485 "TARGET_AVX512F && <round_mode512bit_condition>"
4139 "vfnmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}" 4486 "vfnmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
4188 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}" 4535 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
4189 [(set_attr "type" "ssemuladd") 4536 [(set_attr "type" "ssemuladd")
4190 (set_attr "mode" "<MODE>")]) 4537 (set_attr "mode" "<MODE>")])
4191 4538
4192 (define_insn "*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_1" 4539 (define_insn "*<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name>_bcst_1"
4193 [(set (match_operand:VF_AVX512 0 "register_operand" "=v,v") 4540 [(set (match_operand:VF_AVX512 0 "register_operand" "=v")
4194 (fma:VF_AVX512 4541 (fma:VF_AVX512
4195 (neg:VF_AVX512 4542 (neg:VF_AVX512
4196 (match_operand:VF_AVX512 1 "register_operand" "0,v")) 4543 (match_operand:VF_AVX512 1 "register_operand" "%0"))
4197 (match_operand:VF_AVX512 2 "register_operand" "v,0") 4544 (match_operand:VF_AVX512 2 "register_operand" "v")
4198 (neg:VF_AVX512 4545 (neg:VF_AVX512
4199 (vec_duplicate:VF_AVX512 4546 (vec_duplicate:VF_AVX512
4200 (match_operand:<ssescalarmode> 3 "memory_operand" "m,m")))))] 4547 (match_operand:<ssescalarmode> 3 "memory_operand" "m")))))]
4201 "TARGET_AVX512F && <sd_mask_mode512bit_condition>" 4548 "TARGET_AVX512F && <sd_mask_mode512bit_condition>"
4202 "vfnmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}" 4549 "vfnmsub213<ssemodesuffix>\t{%3<avx512bcst>, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<avx512bcst>}"
4203 [(set_attr "type" "ssemuladd") 4550 [(set_attr "type" "ssemuladd")
4204 (set_attr "mode" "<MODE>")]) 4551 (set_attr "mode" "<MODE>")])
4205 4552
4239 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4586 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4240 (vec_merge:VF_AVX512VL 4587 (vec_merge:VF_AVX512VL
4241 (fma:VF_AVX512VL 4588 (fma:VF_AVX512VL
4242 (neg:VF_AVX512VL 4589 (neg:VF_AVX512VL
4243 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")) 4590 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
4244 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4591 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
4245 (neg:VF_AVX512VL 4592 (neg:VF_AVX512VL
4246 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))) 4593 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>")))
4247 (match_dup 1) 4594 (match_dup 1)
4248 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4595 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4249 "TARGET_AVX512F && <round_mode512bit_condition>" 4596 "TARGET_AVX512F && <round_mode512bit_condition>"
4250 "@ 4597 "@
4251 vfnmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>} 4598 vfnmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
4256 (define_insn "<avx512>_fnmsub_<mode>_mask3<round_name>" 4603 (define_insn "<avx512>_fnmsub_<mode>_mask3<round_name>"
4257 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4604 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4258 (vec_merge:VF_AVX512VL 4605 (vec_merge:VF_AVX512VL
4259 (fma:VF_AVX512VL 4606 (fma:VF_AVX512VL
4260 (neg:VF_AVX512VL 4607 (neg:VF_AVX512VL
4261 (match_operand:VF_AVX512VL 1 "register_operand" "v")) 4608 (match_operand:VF_AVX512VL 1 "<round_nimm_predicate>" "%v"))
4262 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4609 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
4263 (neg:VF_AVX512VL 4610 (neg:VF_AVX512VL
4264 (match_operand:VF_AVX512VL 3 "register_operand" "0"))) 4611 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
4265 (match_dup 3) 4612 (match_dup 3)
4266 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4613 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4267 "TARGET_AVX512F" 4614 "TARGET_AVX512F"
4339 (define_insn "<avx512>_fmaddsub_<mode>_mask<round_name>" 4686 (define_insn "<avx512>_fmaddsub_<mode>_mask<round_name>"
4340 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4687 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4341 (vec_merge:VF_AVX512VL 4688 (vec_merge:VF_AVX512VL
4342 (unspec:VF_AVX512VL 4689 (unspec:VF_AVX512VL
4343 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0") 4690 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
4344 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4691 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
4345 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")] 4692 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>")]
4346 UNSPEC_FMADDSUB) 4693 UNSPEC_FMADDSUB)
4347 (match_dup 1) 4694 (match_dup 1)
4348 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4695 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4349 "TARGET_AVX512F" 4696 "TARGET_AVX512F"
4350 "@ 4697 "@
4356 (define_insn "<avx512>_fmaddsub_<mode>_mask3<round_name>" 4703 (define_insn "<avx512>_fmaddsub_<mode>_mask3<round_name>"
4357 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4704 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4358 (vec_merge:VF_AVX512VL 4705 (vec_merge:VF_AVX512VL
4359 (unspec:VF_AVX512VL 4706 (unspec:VF_AVX512VL
4360 [(match_operand:VF_AVX512VL 1 "register_operand" "v") 4707 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
4361 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4708 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
4362 (match_operand:VF_AVX512VL 3 "register_operand" "0")] 4709 (match_operand:VF_AVX512VL 3 "register_operand" "0")]
4363 UNSPEC_FMADDSUB) 4710 UNSPEC_FMADDSUB)
4364 (match_dup 3) 4711 (match_dup 3)
4365 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4712 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4366 "TARGET_AVX512F" 4713 "TARGET_AVX512F"
4406 (define_insn "<avx512>_fmsubadd_<mode>_mask<round_name>" 4753 (define_insn "<avx512>_fmsubadd_<mode>_mask<round_name>"
4407 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v") 4754 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4408 (vec_merge:VF_AVX512VL 4755 (vec_merge:VF_AVX512VL
4409 (unspec:VF_AVX512VL 4756 (unspec:VF_AVX512VL
4410 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0") 4757 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
4411 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v") 4758 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v")
4412 (neg:VF_AVX512VL 4759 (neg:VF_AVX512VL
4413 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))] 4760 (match_operand:VF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>"))]
4414 UNSPEC_FMADDSUB) 4761 UNSPEC_FMADDSUB)
4415 (match_dup 1) 4762 (match_dup 1)
4416 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))] 4763 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4417 "TARGET_AVX512F" 4764 "TARGET_AVX512F"
4418 "@ 4765 "@
4424 (define_insn "<avx512>_fmsubadd_<mode>_mask3<round_name>" 4771 (define_insn "<avx512>_fmsubadd_<mode>_mask3<round_name>"
4425 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v") 4772 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4426 (vec_merge:VF_AVX512VL 4773 (vec_merge:VF_AVX512VL
4427 (unspec:VF_AVX512VL 4774 (unspec:VF_AVX512VL
4428 [(match_operand:VF_AVX512VL 1 "register_operand" "v") 4775 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
4429 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>") 4776 (match_operand:VF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>")
4430 (neg:VF_AVX512VL 4777 (neg:VF_AVX512VL
4431 (match_operand:VF_AVX512VL 3 "register_operand" "0"))] 4778 (match_operand:VF_AVX512VL 3 "register_operand" "0"))]
4432 UNSPEC_FMADDSUB) 4779 UNSPEC_FMADDSUB)
4433 (match_dup 3) 4780 (match_dup 3)
4434 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))] 4781 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4442 4789
4443 (define_expand "fmai_vmfmadd_<mode><round_name>" 4790 (define_expand "fmai_vmfmadd_<mode><round_name>"
4444 [(set (match_operand:VF_128 0 "register_operand") 4791 [(set (match_operand:VF_128 0 "register_operand")
4445 (vec_merge:VF_128 4792 (vec_merge:VF_128
4446 (fma:VF_128 4793 (fma:VF_128
4447 (match_operand:VF_128 1 "<round_nimm_predicate>") 4794 (match_operand:VF_128 1 "register_operand")
4448 (match_operand:VF_128 2 "<round_nimm_predicate>") 4795 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>")
4449 (match_operand:VF_128 3 "<round_nimm_predicate>")) 4796 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>"))
4450 (match_dup 1) 4797 (match_dup 1)
4451 (const_int 1)))] 4798 (const_int 1)))]
4452 "TARGET_FMA") 4799 "TARGET_FMA")
4453 4800
4454 (define_expand "fmai_vmfmsub_<mode><round_name>" 4801 (define_expand "fmai_vmfmsub_<mode><round_name>"
4455 [(set (match_operand:VF_128 0 "register_operand") 4802 [(set (match_operand:VF_128 0 "register_operand")
4456 (vec_merge:VF_128 4803 (vec_merge:VF_128
4457 (fma:VF_128 4804 (fma:VF_128
4458 (match_operand:VF_128 1 "<round_nimm_predicate>") 4805 (match_operand:VF_128 1 "register_operand")
4459 (match_operand:VF_128 2 "<round_nimm_predicate>") 4806 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>")
4460 (neg:VF_128 4807 (neg:VF_128
4461 (match_operand:VF_128 3 "<round_nimm_predicate>"))) 4808 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>")))
4462 (match_dup 1) 4809 (match_dup 1)
4463 (const_int 1)))] 4810 (const_int 1)))]
4464 "TARGET_FMA") 4811 "TARGET_FMA")
4465 4812
4466 (define_expand "fmai_vmfnmadd_<mode><round_name>" 4813 (define_expand "fmai_vmfnmadd_<mode><round_name>"
4467 [(set (match_operand:VF_128 0 "register_operand") 4814 [(set (match_operand:VF_128 0 "register_operand")
4468 (vec_merge:VF_128 4815 (vec_merge:VF_128
4469 (fma:VF_128 4816 (fma:VF_128
4470 (neg:VF_128 4817 (neg:VF_128
4471 (match_operand:VF_128 2 "<round_nimm_predicate>")) 4818 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>"))
4472 (match_operand:VF_128 1 "<round_nimm_predicate>") 4819 (match_operand:VF_128 1 "register_operand")
4473 (match_operand:VF_128 3 "<round_nimm_predicate>")) 4820 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>"))
4474 (match_dup 1) 4821 (match_dup 1)
4475 (const_int 1)))] 4822 (const_int 1)))]
4476 "TARGET_FMA") 4823 "TARGET_FMA")
4477 4824
4478 (define_expand "fmai_vmfnmsub_<mode><round_name>" 4825 (define_expand "fmai_vmfnmsub_<mode><round_name>"
4479 [(set (match_operand:VF_128 0 "register_operand") 4826 [(set (match_operand:VF_128 0 "register_operand")
4480 (vec_merge:VF_128 4827 (vec_merge:VF_128
4481 (fma:VF_128 4828 (fma:VF_128
4482 (neg:VF_128 4829 (neg:VF_128
4483 (match_operand:VF_128 2 "<round_nimm_predicate>")) 4830 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>"))
4484 (match_operand:VF_128 1 "<round_nimm_predicate>") 4831 (match_operand:VF_128 1 "register_operand")
4485 (neg:VF_128 4832 (neg:VF_128
4486 (match_operand:VF_128 3 "<round_nimm_predicate>"))) 4833 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>")))
4487 (match_dup 1) 4834 (match_dup 1)
4488 (const_int 1)))] 4835 (const_int 1)))]
4489 "TARGET_FMA") 4836 "TARGET_FMA")
4490 4837
4491 (define_insn "*fmai_fmadd_<mode>" 4838 (define_insn "*fmai_fmadd_<mode>"
4492 [(set (match_operand:VF_128 0 "register_operand" "=v,v") 4839 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4493 (vec_merge:VF_128 4840 (vec_merge:VF_128
4494 (fma:VF_128 4841 (fma:VF_128
4495 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0") 4842 (match_operand:VF_128 1 "register_operand" "0,0")
4496 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v") 4843 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>, v")
4497 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>")) 4844 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
4498 (match_dup 1) 4845 (match_dup 1)
4499 (const_int 1)))] 4846 (const_int 1)))]
4500 "TARGET_FMA || TARGET_AVX512F" 4847 "TARGET_FMA || TARGET_AVX512F"
4501 "@ 4848 "@
4502 vfmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>} 4849 vfmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4506 4853
4507 (define_insn "*fmai_fmsub_<mode>" 4854 (define_insn "*fmai_fmsub_<mode>"
4508 [(set (match_operand:VF_128 0 "register_operand" "=v,v") 4855 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4509 (vec_merge:VF_128 4856 (vec_merge:VF_128
4510 (fma:VF_128 4857 (fma:VF_128
4511 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0") 4858 (match_operand:VF_128 1 "register_operand" "0,0")
4512 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v") 4859 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v")
4513 (neg:VF_128 4860 (neg:VF_128
4514 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>"))) 4861 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
4515 (match_dup 1) 4862 (match_dup 1)
4516 (const_int 1)))] 4863 (const_int 1)))]
4517 "TARGET_FMA || TARGET_AVX512F" 4864 "TARGET_FMA || TARGET_AVX512F"
4518 "@ 4865 "@
4519 vfmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>} 4866 vfmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4524 (define_insn "*fmai_fnmadd_<mode><round_name>" 4871 (define_insn "*fmai_fnmadd_<mode><round_name>"
4525 [(set (match_operand:VF_128 0 "register_operand" "=v,v") 4872 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4526 (vec_merge:VF_128 4873 (vec_merge:VF_128
4527 (fma:VF_128 4874 (fma:VF_128
4528 (neg:VF_128 4875 (neg:VF_128
4529 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v")) 4876 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
4530 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0") 4877 (match_operand:VF_128 1 "register_operand" "0,0")
4531 (match_operand:VF_128 3 "<round_nimm_predicate>" "v,<round_constraint>")) 4878 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
4532 (match_dup 1) 4879 (match_dup 1)
4533 (const_int 1)))] 4880 (const_int 1)))]
4534 "TARGET_FMA || TARGET_AVX512F" 4881 "TARGET_FMA || TARGET_AVX512F"
4535 "@ 4882 "@
4536 vfnmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>} 4883 vfnmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4541 (define_insn "*fmai_fnmsub_<mode><round_name>" 4888 (define_insn "*fmai_fnmsub_<mode><round_name>"
4542 [(set (match_operand:VF_128 0 "register_operand" "=v,v") 4889 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4543 (vec_merge:VF_128 4890 (vec_merge:VF_128
4544 (fma:VF_128 4891 (fma:VF_128
4545 (neg:VF_128 4892 (neg:VF_128
4546 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v")) 4893 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
4547 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0") 4894 (match_operand:VF_128 1 "register_operand" "0,0")
4548 (neg:VF_128 4895 (neg:VF_128
4549 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>"))) 4896 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
4550 (match_dup 1) 4897 (match_dup 1)
4551 (const_int 1)))] 4898 (const_int 1)))]
4552 "TARGET_FMA || TARGET_AVX512F" 4899 "TARGET_FMA || TARGET_AVX512F"
4553 "@ 4900 "@
4554 vfnmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>} 4901 vfnmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4555 vfnmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}" 4902 vfnmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
4903 [(set_attr "type" "ssemuladd")
4904 (set_attr "mode" "<MODE>")])
4905
4906 (define_insn "avx512f_vmfmadd_<mode>_mask<round_name>"
4907 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4908 (vec_merge:VF_128
4909 (vec_merge:VF_128
4910 (fma:VF_128
4911 (match_operand:VF_128 1 "register_operand" "0,0")
4912 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v")
4913 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
4914 (match_dup 1)
4915 (match_operand:QI 4 "register_operand" "Yk,Yk"))
4916 (match_dup 1)
4917 (const_int 1)))]
4918 "TARGET_AVX512F"
4919 "@
4920 vfmadd132<ssescalarmodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}
4921 vfmadd213<ssescalarmodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %<iptr>2, %<iptr>3<round_op5>}"
4922 [(set_attr "type" "ssemuladd")
4923 (set_attr "mode" "<MODE>")])
4924
4925 (define_insn "avx512f_vmfmadd_<mode>_mask3<round_name>"
4926 [(set (match_operand:VF_128 0 "register_operand" "=v")
4927 (vec_merge:VF_128
4928 (vec_merge:VF_128
4929 (fma:VF_128
4930 (match_operand:VF_128 1 "<round_nimm_scalar_predicate>" "%v")
4931 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>")
4932 (match_operand:VF_128 3 "register_operand" "0"))
4933 (match_dup 3)
4934 (match_operand:QI 4 "register_operand" "Yk"))
4935 (match_dup 3)
4936 (const_int 1)))]
4937 "TARGET_AVX512F"
4938 "vfmadd231<ssescalarmodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}"
4939 [(set_attr "type" "ssemuladd")
4940 (set_attr "mode" "<MODE>")])
4941
4942 (define_expand "avx512f_vmfmadd_<mode>_maskz<round_expand_name>"
4943 [(match_operand:VF_128 0 "register_operand")
4944 (match_operand:VF_128 1 "<round_expand_nimm_predicate>")
4945 (match_operand:VF_128 2 "<round_expand_nimm_predicate>")
4946 (match_operand:VF_128 3 "<round_expand_nimm_predicate>")
4947 (match_operand:QI 4 "register_operand")]
4948 "TARGET_AVX512F"
4949 {
4950 emit_insn (gen_avx512f_vmfmadd_<mode>_maskz_1<round_expand_name> (
4951 operands[0], operands[1], operands[2], operands[3],
4952 CONST0_RTX (<MODE>mode), operands[4]<round_expand_operand>));
4953 DONE;
4954 })
4955
4956 (define_insn "avx512f_vmfmadd_<mode>_maskz_1<round_name>"
4957 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4958 (vec_merge:VF_128
4959 (vec_merge:VF_128
4960 (fma:VF_128
4961 (match_operand:VF_128 1 "register_operand" "0,0")
4962 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v")
4963 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
4964 (match_operand:VF_128 4 "const0_operand" "C,C")
4965 (match_operand:QI 5 "register_operand" "Yk,Yk"))
4966 (match_dup 1)
4967 (const_int 1)))]
4968 "TARGET_AVX512F"
4969 "@
4970 vfmadd132<ssescalarmodesuffix>\t{<round_op6>%2, %3, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>3, %<iptr>2<round_op6>}
4971 vfmadd213<ssescalarmodesuffix>\t{<round_op6>%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>2, %<iptr>3<round_op6>}"
4972 [(set_attr "type" "ssemuladd")
4973 (set_attr "mode" "<MODE>")])
4974
4975 (define_insn "*avx512f_vmfmsub_<mode>_mask<round_name>"
4976 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4977 (vec_merge:VF_128
4978 (vec_merge:VF_128
4979 (fma:VF_128
4980 (match_operand:VF_128 1 "register_operand" "0,0")
4981 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v")
4982 (neg:VF_128
4983 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
4984 (match_dup 1)
4985 (match_operand:QI 4 "register_operand" "Yk,Yk"))
4986 (match_dup 1)
4987 (const_int 1)))]
4988 "TARGET_AVX512F"
4989 "@
4990 vfmsub132<ssescalarmodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}
4991 vfmsub213<ssescalarmodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %<iptr>2, %<iptr>3<round_op5>}"
4992 [(set_attr "type" "ssemuladd")
4993 (set_attr "mode" "<MODE>")])
4994
4995 (define_insn "avx512f_vmfmsub_<mode>_mask3<round_name>"
4996 [(set (match_operand:VF_128 0 "register_operand" "=v")
4997 (vec_merge:VF_128
4998 (vec_merge:VF_128
4999 (fma:VF_128
5000 (match_operand:VF_128 1 "<round_nimm_scalar_predicate>" "%v")
5001 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>")
5002 (neg:VF_128
5003 (match_operand:VF_128 3 "register_operand" "0")))
5004 (match_dup 3)
5005 (match_operand:QI 4 "register_operand" "Yk"))
5006 (match_dup 3)
5007 (const_int 1)))]
5008 "TARGET_AVX512F"
5009 "vfmsub231<ssescalarmodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}"
5010 [(set_attr "type" "ssemuladd")
5011 (set_attr "mode" "<MODE>")])
5012
5013 (define_insn "*avx512f_vmfmsub_<mode>_maskz_1<round_name>"
5014 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
5015 (vec_merge:VF_128
5016 (vec_merge:VF_128
5017 (fma:VF_128
5018 (match_operand:VF_128 1 "register_operand" "0,0")
5019 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v")
5020 (neg:VF_128
5021 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
5022 (match_operand:VF_128 4 "const0_operand" "C,C")
5023 (match_operand:QI 5 "register_operand" "Yk,Yk"))
5024 (match_dup 1)
5025 (const_int 1)))]
5026 "TARGET_AVX512F"
5027 "@
5028 vfmsub132<ssescalarmodesuffix>\t{<round_op6>%2, %3, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>3, %<iptr>2<round_op6>}
5029 vfmsub213<ssescalarmodesuffix>\t{<round_op6>%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>2, %<iptr>3<round_op6>}"
5030 [(set_attr "type" "ssemuladd")
5031 (set_attr "mode" "<MODE>")])
5032
5033 (define_insn "*avx512f_vmfnmadd_<mode>_mask<round_name>"
5034 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
5035 (vec_merge:VF_128
5036 (vec_merge:VF_128
5037 (fma:VF_128
5038 (neg:VF_128
5039 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
5040 (match_operand:VF_128 1 "register_operand" "0,0")
5041 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
5042 (match_dup 1)
5043 (match_operand:QI 4 "register_operand" "Yk,Yk"))
5044 (match_dup 1)
5045 (const_int 1)))]
5046 "TARGET_AVX512F"
5047 "@
5048 vfnmadd132<ssescalarmodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}
5049 vfnmadd213<ssescalarmodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %<iptr>2, %<iptr>3<round_op5>}"
5050 [(set_attr "type" "ssemuladd")
5051 (set_attr "mode" "<MODE>")])
5052
5053 (define_insn "*avx512f_vmfnmadd_<mode>_mask3<round_name>"
5054 [(set (match_operand:VF_128 0 "register_operand" "=v")
5055 (vec_merge:VF_128
5056 (vec_merge:VF_128
5057 (fma:VF_128
5058 (neg:VF_128
5059 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>"))
5060 (match_operand:VF_128 1 "<round_nimm_scalar_predicate>" "%v")
5061 (match_operand:VF_128 3 "register_operand" "0"))
5062 (match_dup 3)
5063 (match_operand:QI 4 "register_operand" "Yk"))
5064 (match_dup 3)
5065 (const_int 1)))]
5066 "TARGET_AVX512F"
5067 "vfnmadd231<ssescalarmodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}"
5068 [(set_attr "type" "ssemuladd")
5069 (set_attr "mode" "<MODE>")])
5070
5071 (define_insn "*avx512f_vmfnmadd_<mode>_maskz_1<round_name>"
5072 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
5073 (vec_merge:VF_128
5074 (vec_merge:VF_128
5075 (fma:VF_128
5076 (neg:VF_128
5077 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
5078 (match_operand:VF_128 1 "register_operand" "0,0")
5079 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>"))
5080 (match_operand:VF_128 4 "const0_operand" "C,C")
5081 (match_operand:QI 5 "register_operand" "Yk,Yk"))
5082 (match_dup 1)
5083 (const_int 1)))]
5084 "TARGET_AVX512F"
5085 "@
5086 vfnmadd132<ssescalarmodesuffix>\t{<round_op6>%2, %3, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>3, %<iptr>2<round_op6>}
5087 vfnmadd213<ssescalarmodesuffix>\t{<round_op6>%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>2, %<iptr>3<round_op6>}"
5088 [(set_attr "type" "ssemuladd")
5089 (set_attr "mode" "<MODE>")])
5090
5091 (define_insn "*avx512f_vmfnmsub_<mode>_mask<round_name>"
5092 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
5093 (vec_merge:VF_128
5094 (vec_merge:VF_128
5095 (fma:VF_128
5096 (neg:VF_128
5097 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
5098 (match_operand:VF_128 1 "register_operand" "0,0")
5099 (neg:VF_128
5100 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
5101 (match_dup 1)
5102 (match_operand:QI 4 "register_operand" "Yk,Yk"))
5103 (match_dup 1)
5104 (const_int 1)))]
5105 "TARGET_AVX512F"
5106 "@
5107 vfnmsub132<ssescalarmodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}
5108 vfnmsub213<ssescalarmodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %<iptr>2, %<iptr>3<round_op5>}"
5109 [(set_attr "type" "ssemuladd")
5110 (set_attr "mode" "<MODE>")])
5111
5112 (define_insn "*avx512f_vmfnmsub_<mode>_mask3<round_name>"
5113 [(set (match_operand:VF_128 0 "register_operand" "=v")
5114 (vec_merge:VF_128
5115 (vec_merge:VF_128
5116 (fma:VF_128
5117 (neg:VF_128
5118 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>"))
5119 (match_operand:VF_128 1 "<round_nimm_scalar_predicate>" "%v")
5120 (neg:VF_128
5121 (match_operand:VF_128 3 "register_operand" "0")))
5122 (match_dup 3)
5123 (match_operand:QI 4 "register_operand" "Yk"))
5124 (match_dup 3)
5125 (const_int 1)))]
5126 "TARGET_AVX512F"
5127 "vfnmsub231<ssescalarmodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %<iptr>3, %<iptr>2<round_op5>}"
5128 [(set_attr "type" "ssemuladd")
5129 (set_attr "mode" "<MODE>")])
5130
5131 (define_insn "*avx512f_vmfnmsub_<mode>_maskz_1<round_name>"
5132 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
5133 (vec_merge:VF_128
5134 (vec_merge:VF_128
5135 (fma:VF_128
5136 (neg:VF_128
5137 (match_operand:VF_128 2 "<round_nimm_scalar_predicate>" "<round_constraint>,v"))
5138 (match_operand:VF_128 1 "register_operand" "0,0")
5139 (neg:VF_128
5140 (match_operand:VF_128 3 "<round_nimm_scalar_predicate>" "v,<round_constraint>")))
5141 (match_operand:VF_128 4 "const0_operand" "C,C")
5142 (match_operand:QI 5 "register_operand" "Yk,Yk"))
5143 (match_dup 1)
5144 (const_int 1)))]
5145 "TARGET_AVX512F"
5146 "@
5147 vfnmsub132<ssescalarmodesuffix>\t{<round_op6>%2, %3, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>3, %<iptr>2<round_op6>}
5148 vfnmsub213<ssescalarmodesuffix>\t{<round_op6>%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %<iptr>2, %<iptr>3<round_op6>}"
4556 [(set_attr "type" "ssemuladd") 5149 [(set_attr "type" "ssemuladd")
4557 (set_attr "mode" "<MODE>")]) 5150 (set_attr "mode" "<MODE>")])
4558 5151
4559 ;; FMA4 floating point scalar intrinsics. These write the 5152 ;; FMA4 floating point scalar intrinsics. These write the
4560 ;; entire destination register, with the high-order elements zeroed. 5153 ;; entire destination register, with the high-order elements zeroed.
4635 ;; 5228 ;;
4636 ;; Parallel single-precision floating point conversion operations 5229 ;; Parallel single-precision floating point conversion operations
4637 ;; 5230 ;;
4638 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 5231 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4639 5232
4640 (define_insn "sse_cvtpi2ps" 5233 (define_insn_and_split "sse_cvtpi2ps"
4641 [(set (match_operand:V4SF 0 "register_operand" "=x") 5234 [(set (match_operand:V4SF 0 "register_operand" "=x,x,Yv")
4642 (vec_merge:V4SF 5235 (vec_merge:V4SF
4643 (vec_duplicate:V4SF 5236 (vec_duplicate:V4SF
4644 (float:V2SF (match_operand:V2SI 2 "nonimmediate_operand" "ym"))) 5237 (float:V2SF (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yv")))
4645 (match_operand:V4SF 1 "register_operand" "0") 5238 (match_operand:V4SF 1 "register_operand" "0,0,Yv")
4646 (const_int 3)))] 5239 (const_int 3)))
4647 "TARGET_SSE" 5240 (clobber (match_scratch:V4SF 3 "=X,x,Yv"))]
4648 "cvtpi2ps\t{%2, %0|%0, %2}" 5241 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4649 [(set_attr "type" "ssecvt") 5242 "@
5243 cvtpi2ps\t{%2, %0|%0, %2}
5244 #
5245 #"
5246 "TARGET_SSE2 && reload_completed
5247 && SSE_REG_P (operands[2])"
5248 [(const_int 0)]
5249 {
5250 rtx op2 = lowpart_subreg (V4SImode, operands[2],
5251 GET_MODE (operands[2]));
5252 /* Generate SSE2 cvtdq2ps. */
5253 emit_insn (gen_floatv4siv4sf2 (operands[3], op2));
5254
5255 /* Merge operands[3] with operands[0]. */
5256 rtx mask, op1;
5257 if (TARGET_AVX)
5258 {
5259 mask = gen_rtx_PARALLEL (VOIDmode,
5260 gen_rtvec (4, GEN_INT (0), GEN_INT (1),
5261 GEN_INT (6), GEN_INT (7)));
5262 op1 = gen_rtx_VEC_CONCAT (V8SFmode, operands[3], operands[1]);
5263 op2 = gen_rtx_VEC_SELECT (V4SFmode, op1, mask);
5264 emit_insn (gen_rtx_SET (operands[0], op2));
5265 }
5266 else
5267 {
5268 /* NB: SSE can only concatenate OP0 and OP3 to OP0. */
5269 mask = gen_rtx_PARALLEL (VOIDmode,
5270 gen_rtvec (4, GEN_INT (2), GEN_INT (3),
5271 GEN_INT (4), GEN_INT (5)));
5272 op1 = gen_rtx_VEC_CONCAT (V8SFmode, operands[0], operands[3]);
5273 op2 = gen_rtx_VEC_SELECT (V4SFmode, op1, mask);
5274 emit_insn (gen_rtx_SET (operands[0], op2));
5275
5276 /* Swap bits 0:63 with bits 64:127. */
5277 mask = gen_rtx_PARALLEL (VOIDmode,
5278 gen_rtvec (4, GEN_INT (2), GEN_INT (3),
5279 GEN_INT (0), GEN_INT (1)));
5280 rtx dest = lowpart_subreg (V4SImode, operands[0],
5281 GET_MODE (operands[0]));
5282 op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
5283 emit_insn (gen_rtx_SET (dest, op1));
5284 }
5285 DONE;
5286 }
5287 [(set_attr "mmx_isa" "native,sse_noavx,avx")
5288 (set_attr "type" "ssecvt")
4650 (set_attr "mode" "V4SF")]) 5289 (set_attr "mode" "V4SF")])
4651 5290
4652 (define_insn "sse_cvtps2pi" 5291 (define_insn "sse_cvtps2pi"
4653 [(set (match_operand:V2SI 0 "register_operand" "=y") 5292 [(set (match_operand:V2SI 0 "register_operand" "=y,Yv")
4654 (vec_select:V2SI 5293 (vec_select:V2SI
4655 (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")] 5294 (unspec:V4SI [(match_operand:V4SF 1 "register_mmxmem_operand" "xm,YvBm")]
4656 UNSPEC_FIX_NOTRUNC) 5295 UNSPEC_FIX_NOTRUNC)
4657 (parallel [(const_int 0) (const_int 1)])))] 5296 (parallel [(const_int 0) (const_int 1)])))]
4658 "TARGET_SSE" 5297 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4659 "cvtps2pi\t{%1, %0|%0, %q1}" 5298 "@
4660 [(set_attr "type" "ssecvt") 5299 cvtps2pi\t{%1, %0|%0, %q1}
4661 (set_attr "unit" "mmx") 5300 %vcvtps2dq\t{%1, %0|%0, %1}"
5301 [(set_attr "isa" "*,sse2")
5302 (set_attr "mmx_isa" "native,*")
5303 (set_attr "type" "ssecvt")
5304 (set_attr "unit" "mmx,*")
4662 (set_attr "mode" "DI")]) 5305 (set_attr "mode" "DI")])
4663 5306
4664 (define_insn "sse_cvttps2pi" 5307 (define_insn "sse_cvttps2pi"
4665 [(set (match_operand:V2SI 0 "register_operand" "=y") 5308 [(set (match_operand:V2SI 0 "register_operand" "=y,Yv")
4666 (vec_select:V2SI 5309 (vec_select:V2SI
4667 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm")) 5310 (fix:V4SI (match_operand:V4SF 1 "register_mmxmem_operand" "xm,YvBm"))
4668 (parallel [(const_int 0) (const_int 1)])))] 5311 (parallel [(const_int 0) (const_int 1)])))]
4669 "TARGET_SSE" 5312 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSE"
4670 "cvttps2pi\t{%1, %0|%0, %q1}" 5313 "@
4671 [(set_attr "type" "ssecvt") 5314 cvttps2pi\t{%1, %0|%0, %q1}
4672 (set_attr "unit" "mmx") 5315 %vcvttps2dq\t{%1, %0|%0, %1}"
5316 [(set_attr "isa" "*,sse2")
5317 (set_attr "mmx_isa" "native,*")
5318 (set_attr "type" "ssecvt")
5319 (set_attr "unit" "mmx,*")
4673 (set_attr "prefix_rep" "0") 5320 (set_attr "prefix_rep" "0")
4674 (set_attr "mode" "SF")]) 5321 (set_attr "mode" "SF")])
4675 5322
4676 (define_insn "sse_cvtsi2ss<rex64namesuffix><round_name>" 5323 (define_insn "sse_cvtsi2ss<rex64namesuffix><round_name>"
4677 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v") 5324 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
4726 (define_insn "sse_cvtss2si<rex64namesuffix>_2" 5373 (define_insn "sse_cvtss2si<rex64namesuffix>_2"
4727 [(set (match_operand:SWI48 0 "register_operand" "=r,r") 5374 [(set (match_operand:SWI48 0 "register_operand" "=r,r")
4728 (unspec:SWI48 [(match_operand:SF 1 "nonimmediate_operand" "v,m")] 5375 (unspec:SWI48 [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
4729 UNSPEC_FIX_NOTRUNC))] 5376 UNSPEC_FIX_NOTRUNC))]
4730 "TARGET_SSE" 5377 "TARGET_SSE"
4731 "%vcvtss2si<rex64suffix>\t{%1, %0|%0, %k1}" 5378 "%vcvtss2si<rex64suffix>\t{%1, %0|%0, %1}"
4732 [(set_attr "type" "sseicvt") 5379 [(set_attr "type" "sseicvt")
4733 (set_attr "athlon_decode" "double,vector") 5380 (set_attr "athlon_decode" "double,vector")
4734 (set_attr "amdfam10_decode" "double,double") 5381 (set_attr "amdfam10_decode" "double,double")
4735 (set_attr "bdver1_decode" "double,double") 5382 (set_attr "bdver1_decode" "double,double")
4736 (set_attr "prefix_rep" "1") 5383 (set_attr "prefix_rep" "1")
4756 (define_insn "cvtusi2<ssescalarmodesuffix>32<round_name>" 5403 (define_insn "cvtusi2<ssescalarmodesuffix>32<round_name>"
4757 [(set (match_operand:VF_128 0 "register_operand" "=v") 5404 [(set (match_operand:VF_128 0 "register_operand" "=v")
4758 (vec_merge:VF_128 5405 (vec_merge:VF_128
4759 (vec_duplicate:VF_128 5406 (vec_duplicate:VF_128
4760 (unsigned_float:<ssescalarmode> 5407 (unsigned_float:<ssescalarmode>
4761 (match_operand:SI 2 "<round_nimm_predicate>" "<round_constraint3>"))) 5408 (match_operand:SI 2 "<round_nimm_scalar_predicate>" "<round_constraint3>")))
4762 (match_operand:VF_128 1 "register_operand" "v") 5409 (match_operand:VF_128 1 "register_operand" "v")
4763 (const_int 1)))] 5410 (const_int 1)))]
4764 "TARGET_AVX512F && <round_modev4sf_condition>" 5411 "TARGET_AVX512F && <round_modev4sf_condition>"
4765 "vcvtusi2<ssescalarmodesuffix>\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}" 5412 "vcvtusi2<ssescalarmodesuffix>{l}\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4766 [(set_attr "type" "sseicvt") 5413 [(set_attr "type" "sseicvt")
4767 (set_attr "prefix" "evex") 5414 (set_attr "prefix" "evex")
4768 (set_attr "mode" "<ssescalarmode>")]) 5415 (set_attr "mode" "<ssescalarmode>")])
4769 5416
4770 (define_insn "cvtusi2<ssescalarmodesuffix>64<round_name>" 5417 (define_insn "cvtusi2<ssescalarmodesuffix>64<round_name>"
4771 [(set (match_operand:VF_128 0 "register_operand" "=v") 5418 [(set (match_operand:VF_128 0 "register_operand" "=v")
4772 (vec_merge:VF_128 5419 (vec_merge:VF_128
4773 (vec_duplicate:VF_128 5420 (vec_duplicate:VF_128
4774 (unsigned_float:<ssescalarmode> 5421 (unsigned_float:<ssescalarmode>
4775 (match_operand:DI 2 "<round_nimm_predicate>" "<round_constraint3>"))) 5422 (match_operand:DI 2 "<round_nimm_scalar_predicate>" "<round_constraint3>")))
4776 (match_operand:VF_128 1 "register_operand" "v") 5423 (match_operand:VF_128 1 "register_operand" "v")
4777 (const_int 1)))] 5424 (const_int 1)))]
4778 "TARGET_AVX512F && TARGET_64BIT" 5425 "TARGET_AVX512F && TARGET_64BIT"
4779 "vcvtusi2<ssescalarmodesuffix>{q}\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}" 5426 "vcvtusi2<ssescalarmodesuffix>{q}\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4780 [(set_attr "type" "sseicvt") 5427 [(set_attr "type" "sseicvt")
4977 ;; Parallel double-precision floating point conversion operations 5624 ;; Parallel double-precision floating point conversion operations
4978 ;; 5625 ;;
4979 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 5626 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4980 5627
4981 (define_insn "sse2_cvtpi2pd" 5628 (define_insn "sse2_cvtpi2pd"
4982 [(set (match_operand:V2DF 0 "register_operand" "=x,x") 5629 [(set (match_operand:V2DF 0 "register_operand" "=v,x")
4983 (float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "y,m")))] 5630 (float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "vBm,?!y")))]
4984 "TARGET_SSE2" 5631 "TARGET_SSE2"
4985 "cvtpi2pd\t{%1, %0|%0, %1}" 5632 "@
4986 [(set_attr "type" "ssecvt") 5633 %vcvtdq2pd\t{%1, %0|%0, %1}
4987 (set_attr "unit" "mmx,*") 5634 cvtpi2pd\t{%1, %0|%0, %1}"
4988 (set_attr "prefix_data16" "1,*") 5635 [(set_attr "mmx_isa" "*,native")
5636 (set_attr "type" "ssecvt")
5637 (set_attr "unit" "*,mmx")
5638 (set_attr "prefix_data16" "*,1")
5639 (set_attr "prefix" "maybe_vex,*")
4989 (set_attr "mode" "V2DF")]) 5640 (set_attr "mode" "V2DF")])
4990 5641
4991 (define_insn "sse2_cvtpd2pi" 5642 (define_insn "sse2_cvtpd2pi"
4992 [(set (match_operand:V2SI 0 "register_operand" "=y") 5643 [(set (match_operand:V2SI 0 "register_operand" "=v,?!y")
4993 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")] 5644 (unspec:V2SI [(match_operand:V2DF 1 "vector_operand" "vBm,xBm")]
4994 UNSPEC_FIX_NOTRUNC))] 5645 UNSPEC_FIX_NOTRUNC))]
4995 "TARGET_SSE2" 5646 "TARGET_SSE2"
4996 "cvtpd2pi\t{%1, %0|%0, %1}" 5647 "@
4997 [(set_attr "type" "ssecvt") 5648 * return TARGET_AVX ? \"vcvtpd2dq{x}\t{%1, %0|%0, %1}\" : \"cvtpd2dq\t{%1, %0|%0, %1}\";
4998 (set_attr "unit" "mmx") 5649 cvtpd2pi\t{%1, %0|%0, %1}"
5650 [(set_attr "mmx_isa" "*,native")
5651 (set_attr "type" "ssecvt")
5652 (set_attr "unit" "*,mmx")
5653 (set_attr "amdfam10_decode" "double")
5654 (set_attr "athlon_decode" "vector")
4999 (set_attr "bdver1_decode" "double") 5655 (set_attr "bdver1_decode" "double")
5000 (set_attr "btver2_decode" "direct") 5656 (set_attr "prefix_data16" "*,1")
5001 (set_attr "prefix_data16" "1") 5657 (set_attr "prefix" "maybe_vex,*")
5002 (set_attr "mode" "DI")]) 5658 (set_attr "mode" "TI")])
5003 5659
5004 (define_insn "sse2_cvttpd2pi" 5660 (define_insn "sse2_cvttpd2pi"
5005 [(set (match_operand:V2SI 0 "register_operand" "=y") 5661 [(set (match_operand:V2SI 0 "register_operand" "=v,?!y")
5006 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "xm")))] 5662 (fix:V2SI (match_operand:V2DF 1 "vector_operand" "vBm,xBm")))]
5007 "TARGET_SSE2" 5663 "TARGET_SSE2"
5008 "cvttpd2pi\t{%1, %0|%0, %1}" 5664 "@
5009 [(set_attr "type" "ssecvt") 5665 * return TARGET_AVX ? \"vcvttpd2dq{x}\t{%1, %0|%0, %1}\" : \"cvttpd2dq\t{%1, %0|%0, %1}\";
5010 (set_attr "unit" "mmx") 5666 cvttpd2pi\t{%1, %0|%0, %1}"
5667 [(set_attr "mmx_isa" "*,native")
5668 (set_attr "type" "ssecvt")
5669 (set_attr "unit" "*,mmx")
5670 (set_attr "amdfam10_decode" "double")
5671 (set_attr "athlon_decode" "vector")
5011 (set_attr "bdver1_decode" "double") 5672 (set_attr "bdver1_decode" "double")
5012 (set_attr "prefix_data16" "1") 5673 (set_attr "prefix_data16" "*,1")
5674 (set_attr "prefix" "maybe_vex,*")
5013 (set_attr "mode" "TI")]) 5675 (set_attr "mode" "TI")])
5014 5676
5015 (define_insn "sse2_cvtsi2sd" 5677 (define_insn "sse2_cvtsi2sd"
5016 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v") 5678 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
5017 (vec_merge:V2DF 5679 (vec_merge:V2DF
5019 (float:DF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm"))) 5681 (float:DF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm")))
5020 (match_operand:V2DF 1 "register_operand" "0,0,v") 5682 (match_operand:V2DF 1 "register_operand" "0,0,v")
5021 (const_int 1)))] 5683 (const_int 1)))]
5022 "TARGET_SSE2" 5684 "TARGET_SSE2"
5023 "@ 5685 "@
5024 cvtsi2sd\t{%2, %0|%0, %2} 5686 cvtsi2sd{l}\t{%2, %0|%0, %2}
5025 cvtsi2sd\t{%2, %0|%0, %2} 5687 cvtsi2sd{l}\t{%2, %0|%0, %2}
5026 vcvtsi2sd\t{%2, %1, %0|%0, %1, %2}" 5688 vcvtsi2sd{l}\t{%2, %1, %0|%0, %1, %2}"
5027 [(set_attr "isa" "noavx,noavx,avx") 5689 [(set_attr "isa" "noavx,noavx,avx")
5028 (set_attr "type" "sseicvt") 5690 (set_attr "type" "sseicvt")
5029 (set_attr "athlon_decode" "double,direct,*") 5691 (set_attr "athlon_decode" "double,direct,*")
5030 (set_attr "amdfam10_decode" "vector,double,*") 5692 (set_attr "amdfam10_decode" "vector,double,*")
5031 (set_attr "bdver1_decode" "double,direct,*") 5693 (set_attr "bdver1_decode" "double,direct,*")
5041 (float:DF (match_operand:DI 2 "<round_nimm_scalar_predicate>" "r,m,<round_constraint3>"))) 5703 (float:DF (match_operand:DI 2 "<round_nimm_scalar_predicate>" "r,m,<round_constraint3>")))
5042 (match_operand:V2DF 1 "register_operand" "0,0,v") 5704 (match_operand:V2DF 1 "register_operand" "0,0,v")
5043 (const_int 1)))] 5705 (const_int 1)))]
5044 "TARGET_SSE2 && TARGET_64BIT" 5706 "TARGET_SSE2 && TARGET_64BIT"
5045 "@ 5707 "@
5046 cvtsi2sdq\t{%2, %0|%0, %2} 5708 cvtsi2sd{q}\t{%2, %0|%0, %2}
5047 cvtsi2sdq\t{%2, %0|%0, %2} 5709 cvtsi2sd{q}\t{%2, %0|%0, %2}
5048 vcvtsi2sdq\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}" 5710 vcvtsi2sd{q}\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
5049 [(set_attr "isa" "noavx,noavx,avx") 5711 [(set_attr "isa" "noavx,noavx,avx")
5050 (set_attr "type" "sseicvt") 5712 (set_attr "type" "sseicvt")
5051 (set_attr "athlon_decode" "double,direct,*") 5713 (set_attr "athlon_decode" "double,direct,*")
5052 (set_attr "amdfam10_decode" "vector,double,*") 5714 (set_attr "amdfam10_decode" "vector,double,*")
5053 (set_attr "bdver1_decode" "double,direct,*") 5715 (set_attr "bdver1_decode" "double,direct,*")
5201 "vcvt<floatsuffix>qq2ps<qq2pssuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}" 5863 "vcvt<floatsuffix>qq2ps<qq2pssuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5202 [(set_attr "type" "ssecvt") 5864 [(set_attr "type" "ssecvt")
5203 (set_attr "prefix" "evex") 5865 (set_attr "prefix" "evex")
5204 (set_attr "mode" "<MODE>")]) 5866 (set_attr "mode" "<MODE>")])
5205 5867
5206 (define_insn "float<floatunssuffix>v2div2sf2" 5868 (define_expand "float<floatunssuffix>v2div2sf2"
5207 [(set (match_operand:V4SF 0 "register_operand" "=v") 5869 [(set (match_operand:V4SF 0 "register_operand" "=v")
5208 (vec_concat:V4SF 5870 (vec_concat:V4SF
5209 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm")) 5871 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5210 (const_vector:V2SF [(const_int 0) (const_int 0)])))] 5872 (match_dup 2)))]
5873 "TARGET_AVX512DQ && TARGET_AVX512VL"
5874 "operands[2] = CONST0_RTX (V2SFmode);")
5875
5876 (define_insn "*float<floatunssuffix>v2div2sf2"
5877 [(set (match_operand:V4SF 0 "register_operand" "=v")
5878 (vec_concat:V4SF
5879 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5880 (match_operand:V2SF 2 "const0_operand" "C")))]
5211 "TARGET_AVX512DQ && TARGET_AVX512VL" 5881 "TARGET_AVX512DQ && TARGET_AVX512VL"
5212 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0|%0, %1}" 5882 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0|%0, %1}"
5213 [(set_attr "type" "ssecvt") 5883 [(set_attr "type" "ssecvt")
5214 (set_attr "prefix" "evex") 5884 (set_attr "prefix" "evex")
5215 (set_attr "mode" "V4SF")]) 5885 (set_attr "mode" "V4SF")])
5239 emit_insn (gen_avx_vec_concat<vpckfloat_concat_mode> (operands[0], 5909 emit_insn (gen_avx_vec_concat<vpckfloat_concat_mode> (operands[0],
5240 r1, r2)); 5910 r1, r2));
5241 DONE; 5911 DONE;
5242 }) 5912 })
5243 5913
5244 (define_insn "float<floatunssuffix>v2div2sf2_mask" 5914 (define_expand "float<floatunssuffix>v2div2sf2_mask"
5245 [(set (match_operand:V4SF 0 "register_operand" "=v") 5915 [(set (match_operand:V4SF 0 "register_operand" "=v")
5246 (vec_concat:V4SF 5916 (vec_concat:V4SF
5247 (vec_merge:V2SF 5917 (vec_merge:V2SF
5248 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm")) 5918 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5249 (vec_select:V2SF 5919 (vec_select:V2SF
5250 (match_operand:V4SF 2 "nonimm_or_0_operand" "0C") 5920 (match_operand:V4SF 2 "nonimm_or_0_operand" "0C")
5251 (parallel [(const_int 0) (const_int 1)])) 5921 (parallel [(const_int 0) (const_int 1)]))
5252 (match_operand:QI 3 "register_operand" "Yk")) 5922 (match_operand:QI 3 "register_operand" "Yk"))
5253 (const_vector:V2SF [(const_int 0) (const_int 0)])))] 5923 (match_dup 4)))]
5924 "TARGET_AVX512DQ && TARGET_AVX512VL"
5925 "operands[4] = CONST0_RTX (V2SFmode);")
5926
5927 (define_insn "*float<floatunssuffix>v2div2sf2_mask"
5928 [(set (match_operand:V4SF 0 "register_operand" "=v")
5929 (vec_concat:V4SF
5930 (vec_merge:V2SF
5931 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5932 (vec_select:V2SF
5933 (match_operand:V4SF 2 "nonimm_or_0_operand" "0C")
5934 (parallel [(const_int 0) (const_int 1)]))
5935 (match_operand:QI 3 "register_operand" "Yk"))
5936 (match_operand:V2SF 4 "const0_operand" "C")))]
5254 "TARGET_AVX512DQ && TARGET_AVX512VL" 5937 "TARGET_AVX512DQ && TARGET_AVX512VL"
5255 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}" 5938 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
5256 [(set_attr "type" "ssecvt") 5939 [(set_attr "type" "ssecvt")
5257 (set_attr "prefix" "evex") 5940 (set_attr "prefix" "evex")
5258 (set_attr "mode" "V4SF")]) 5941 (set_attr "mode" "V4SF")])
5261 [(set (match_operand:V4SF 0 "register_operand" "=v") 5944 [(set (match_operand:V4SF 0 "register_operand" "=v")
5262 (vec_concat:V4SF 5945 (vec_concat:V4SF
5263 (vec_merge:V2SF 5946 (vec_merge:V2SF
5264 (any_float:V2SF (match_operand:V2DI 1 5947 (any_float:V2SF (match_operand:V2DI 1
5265 "nonimmediate_operand" "vm")) 5948 "nonimmediate_operand" "vm"))
5266 (const_vector:V2SF [(const_int 0) (const_int 0)]) 5949 (match_operand:V2SF 3 "const0_operand" "C")
5267 (match_operand:QI 2 "register_operand" "Yk")) 5950 (match_operand:QI 2 "register_operand" "Yk"))
5268 (const_vector:V2SF [(const_int 0) (const_int 0)])))] 5951 (match_operand:V2SF 4 "const0_operand" "C")))]
5269 "TARGET_AVX512DQ && TARGET_AVX512VL" 5952 "TARGET_AVX512DQ && TARGET_AVX512VL"
5270 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}" 5953 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
5271 [(set_attr "type" "ssecvt") 5954 [(set_attr "type" "ssecvt")
5272 (set_attr "prefix" "evex") 5955 (set_attr "prefix" "evex")
5273 (set_attr "mode" "V4SF")]) 5956 (set_attr "mode" "V4SF")])
5375 [(set_attr "type" "ssecvt") 6058 [(set_attr "type" "ssecvt")
5376 (set_attr "prefix" "vex") 6059 (set_attr "prefix" "vex")
5377 (set_attr "btver2_decode" "vector") 6060 (set_attr "btver2_decode" "vector")
5378 (set_attr "mode" "OI")]) 6061 (set_attr "mode" "OI")])
5379 6062
5380 (define_insn "sse2_cvtpd2dq<mask_name>" 6063 (define_insn "sse2_cvtpd2dq"
5381 [(set (match_operand:V4SI 0 "register_operand" "=v") 6064 [(set (match_operand:V4SI 0 "register_operand" "=v")
5382 (vec_concat:V4SI 6065 (vec_concat:V4SI
5383 (unspec:V2SI [(match_operand:V2DF 1 "vector_operand" "vBm")] 6066 (unspec:V2SI [(match_operand:V2DF 1 "vector_operand" "vBm")]
5384 UNSPEC_FIX_NOTRUNC) 6067 UNSPEC_FIX_NOTRUNC)
5385 (const_vector:V2SI [(const_int 0) (const_int 0)])))] 6068 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5386 "TARGET_SSE2 && <mask_avx512vl_condition>" 6069 "TARGET_SSE2"
5387 { 6070 {
5388 if (TARGET_AVX) 6071 if (TARGET_AVX)
5389 return "vcvtpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"; 6072 return "vcvtpd2dq{x}\t{%1, %0|%0, %1}";
5390 else 6073 else
5391 return "cvtpd2dq\t{%1, %0|%0, %1}"; 6074 return "cvtpd2dq\t{%1, %0|%0, %1}";
5392 } 6075 }
5393 [(set_attr "type" "ssecvt") 6076 [(set_attr "type" "ssecvt")
5394 (set_attr "prefix_rep" "1") 6077 (set_attr "prefix_rep" "1")
5397 (set_attr "mode" "TI") 6080 (set_attr "mode" "TI")
5398 (set_attr "amdfam10_decode" "double") 6081 (set_attr "amdfam10_decode" "double")
5399 (set_attr "athlon_decode" "vector") 6082 (set_attr "athlon_decode" "vector")
5400 (set_attr "bdver1_decode" "double")]) 6083 (set_attr "bdver1_decode" "double")])
5401 6084
6085 (define_insn "sse2_cvtpd2dq_mask"
6086 [(set (match_operand:V4SI 0 "register_operand" "=v")
6087 (vec_concat:V4SI
6088 (vec_merge:V2SI
6089 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
6090 UNSPEC_FIX_NOTRUNC)
6091 (vec_select:V2SI
6092 (match_operand:V4SI 2 "nonimm_or_0_operand" "0C")
6093 (parallel [(const_int 0) (const_int 1)]))
6094 (match_operand:QI 3 "register_operand" "Yk"))
6095 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6096 "TARGET_AVX512VL"
6097 "vcvtpd2dq{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
6098 [(set_attr "type" "ssecvt")
6099 (set_attr "prefix" "evex")
6100 (set_attr "mode" "TI")])
6101
6102 (define_insn "*sse2_cvtpd2dq_mask_1"
6103 [(set (match_operand:V4SI 0 "register_operand" "=v")
6104 (vec_concat:V4SI
6105 (vec_merge:V2SI
6106 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
6107 UNSPEC_FIX_NOTRUNC)
6108 (const_vector:V2SI [(const_int 0) (const_int 0)])
6109 (match_operand:QI 2 "register_operand" "Yk"))
6110 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6111 "TARGET_AVX512VL"
6112 "vcvtpd2dq{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
6113 [(set_attr "type" "ssecvt")
6114 (set_attr "prefix" "evex")
6115 (set_attr "mode" "TI")])
6116
5402 ;; For ufix_notrunc* insn patterns 6117 ;; For ufix_notrunc* insn patterns
5403 (define_mode_attr pd2udqsuff 6118 (define_mode_attr pd2udqsuff
5404 [(V8DF "") (V4DF "{y}")]) 6119 [(V8DF "") (V4DF "{y}")])
5405 6120
5406 (define_insn "ufix_notrunc<mode><si2dfmodelower>2<mask_name><round_name>" 6121 (define_insn "ufix_notrunc<mode><si2dfmodelower>2<mask_name><round_name>"
5412 "vcvtpd2udq<pd2udqsuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}" 6127 "vcvtpd2udq<pd2udqsuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5413 [(set_attr "type" "ssecvt") 6128 [(set_attr "type" "ssecvt")
5414 (set_attr "prefix" "evex") 6129 (set_attr "prefix" "evex")
5415 (set_attr "mode" "<sseinsnmode>")]) 6130 (set_attr "mode" "<sseinsnmode>")])
5416 6131
5417 (define_insn "ufix_notruncv2dfv2si2<mask_name>" 6132 (define_insn "ufix_notruncv2dfv2si2"
5418 [(set (match_operand:V4SI 0 "register_operand" "=v") 6133 [(set (match_operand:V4SI 0 "register_operand" "=v")
5419 (vec_concat:V4SI 6134 (vec_concat:V4SI
5420 (unspec:V2SI 6135 (unspec:V2SI
5421 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")] 6136 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
5422 UNSPEC_UNSIGNED_FIX_NOTRUNC) 6137 UNSPEC_UNSIGNED_FIX_NOTRUNC)
5423 (const_vector:V2SI [(const_int 0) (const_int 0)])))] 6138 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5424 "TARGET_AVX512VL" 6139 "TARGET_AVX512VL"
5425 "vcvtpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}" 6140 "vcvtpd2udq{x}\t{%1, %0|%0, %1}"
6141 [(set_attr "type" "ssecvt")
6142 (set_attr "prefix" "evex")
6143 (set_attr "mode" "TI")])
6144
6145 (define_insn "ufix_notruncv2dfv2si2_mask"
6146 [(set (match_operand:V4SI 0 "register_operand" "=v")
6147 (vec_concat:V4SI
6148 (vec_merge:V2SI
6149 (unspec:V2SI
6150 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
6151 UNSPEC_UNSIGNED_FIX_NOTRUNC)
6152 (vec_select:V2SI
6153 (match_operand:V4SI 2 "nonimm_or_0_operand" "0C")
6154 (parallel [(const_int 0) (const_int 1)]))
6155 (match_operand:QI 3 "register_operand" "Yk"))
6156 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6157 "TARGET_AVX512VL"
6158 "vcvtpd2udq{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
6159 [(set_attr "type" "ssecvt")
6160 (set_attr "prefix" "evex")
6161 (set_attr "mode" "TI")])
6162
6163 (define_insn "*ufix_notruncv2dfv2si2_mask_1"
6164 [(set (match_operand:V4SI 0 "register_operand" "=v")
6165 (vec_concat:V4SI
6166 (vec_merge:V2SI
6167 (unspec:V2SI
6168 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
6169 UNSPEC_UNSIGNED_FIX_NOTRUNC)
6170 (const_vector:V2SI [(const_int 0) (const_int 0)])
6171 (match_operand:QI 2 "register_operand" "Yk"))
6172 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6173 "TARGET_AVX512VL"
6174 "vcvtpd2udq{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
5426 [(set_attr "type" "ssecvt") 6175 [(set_attr "type" "ssecvt")
5427 (set_attr "prefix" "evex") 6176 (set_attr "prefix" "evex")
5428 (set_attr "mode" "TI")]) 6177 (set_attr "mode" "TI")])
5429 6178
5430 (define_insn "fix<fixunssuffix>_truncv8dfv8si2<mask_name><round_saeonly_name>" 6179 (define_insn "fix<fixunssuffix>_truncv8dfv8si2<mask_name><round_saeonly_name>"
5435 "vcvttpd2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}" 6184 "vcvttpd2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
5436 [(set_attr "type" "ssecvt") 6185 [(set_attr "type" "ssecvt")
5437 (set_attr "prefix" "evex") 6186 (set_attr "prefix" "evex")
5438 (set_attr "mode" "OI")]) 6187 (set_attr "mode" "OI")])
5439 6188
5440 (define_insn "ufix_truncv2dfv2si2<mask_name>" 6189 (define_insn "ufix_truncv2dfv2si2"
5441 [(set (match_operand:V4SI 0 "register_operand" "=v") 6190 [(set (match_operand:V4SI 0 "register_operand" "=v")
5442 (vec_concat:V4SI 6191 (vec_concat:V4SI
5443 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm")) 6192 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
5444 (const_vector:V2SI [(const_int 0) (const_int 0)])))] 6193 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5445 "TARGET_AVX512VL" 6194 "TARGET_AVX512VL"
5446 "vcvttpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}" 6195 "vcvttpd2udq{x}\t{%1, %0|%0, %1}"
6196 [(set_attr "type" "ssecvt")
6197 (set_attr "prefix" "evex")
6198 (set_attr "mode" "TI")])
6199
6200 (define_insn "ufix_truncv2dfv2si2_mask"
6201 [(set (match_operand:V4SI 0 "register_operand" "=v")
6202 (vec_concat:V4SI
6203 (vec_merge:V2SI
6204 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6205 (vec_select:V2SI
6206 (match_operand:V4SI 2 "nonimm_or_0_operand" "0C")
6207 (parallel [(const_int 0) (const_int 1)]))
6208 (match_operand:QI 3 "register_operand" "Yk"))
6209 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6210 "TARGET_AVX512VL"
6211 "vcvttpd2udq{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
6212 [(set_attr "type" "ssecvt")
6213 (set_attr "prefix" "evex")
6214 (set_attr "mode" "TI")])
6215
6216 (define_insn "*ufix_truncv2dfv2si2_mask_1"
6217 [(set (match_operand:V4SI 0 "register_operand" "=v")
6218 (vec_concat:V4SI
6219 (vec_merge:V2SI
6220 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6221 (const_vector:V2SI [(const_int 0) (const_int 0)])
6222 (match_operand:QI 2 "register_operand" "Yk"))
6223 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6224 "TARGET_AVX512VL"
6225 "vcvttpd2udq{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
5447 [(set_attr "type" "ssecvt") 6226 [(set_attr "type" "ssecvt")
5448 (set_attr "prefix" "evex") 6227 (set_attr "prefix" "evex")
5449 (set_attr "mode" "TI")]) 6228 (set_attr "mode" "TI")])
5450 6229
5451 (define_insn "fix_truncv4dfv4si2<mask_name>" 6230 (define_insn "fix_truncv4dfv4si2<mask_name>"
5586 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand")) 6365 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand"))
5587 (match_dup 2)))] 6366 (match_dup 2)))]
5588 "TARGET_AVX" 6367 "TARGET_AVX"
5589 "operands[2] = CONST0_RTX (V4SImode);") 6368 "operands[2] = CONST0_RTX (V4SImode);")
5590 6369
5591 (define_insn "sse2_cvttpd2dq<mask_name>" 6370 (define_insn "sse2_cvttpd2dq"
5592 [(set (match_operand:V4SI 0 "register_operand" "=v") 6371 [(set (match_operand:V4SI 0 "register_operand" "=v")
5593 (vec_concat:V4SI 6372 (vec_concat:V4SI
5594 (fix:V2SI (match_operand:V2DF 1 "vector_operand" "vBm")) 6373 (fix:V2SI (match_operand:V2DF 1 "vector_operand" "vBm"))
5595 (const_vector:V2SI [(const_int 0) (const_int 0)])))] 6374 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5596 "TARGET_SSE2 && <mask_avx512vl_condition>" 6375 "TARGET_SSE2"
5597 { 6376 {
5598 if (TARGET_AVX) 6377 if (TARGET_AVX)
5599 return "vcvttpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"; 6378 return "vcvttpd2dq{x}\t{%1, %0|%0, %1}";
5600 else 6379 else
5601 return "cvttpd2dq\t{%1, %0|%0, %1}"; 6380 return "cvttpd2dq\t{%1, %0|%0, %1}";
5602 } 6381 }
5603 [(set_attr "type" "ssecvt") 6382 [(set_attr "type" "ssecvt")
5604 (set_attr "amdfam10_decode" "double") 6383 (set_attr "amdfam10_decode" "double")
5605 (set_attr "athlon_decode" "vector") 6384 (set_attr "athlon_decode" "vector")
5606 (set_attr "bdver1_decode" "double") 6385 (set_attr "bdver1_decode" "double")
5607 (set_attr "prefix" "maybe_vex") 6386 (set_attr "prefix" "maybe_vex")
6387 (set_attr "mode" "TI")])
6388
6389 (define_insn "sse2_cvttpd2dq_mask"
6390 [(set (match_operand:V4SI 0 "register_operand" "=v")
6391 (vec_concat:V4SI
6392 (vec_merge:V2SI
6393 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6394 (vec_select:V2SI
6395 (match_operand:V4SI 2 "nonimm_or_0_operand" "0C")
6396 (parallel [(const_int 0) (const_int 1)]))
6397 (match_operand:QI 3 "register_operand" "Yk"))
6398 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6399 "TARGET_AVX512VL"
6400 "vcvttpd2dq{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
6401 [(set_attr "type" "ssecvt")
6402 (set_attr "prefix" "evex")
6403 (set_attr "mode" "TI")])
6404
6405 (define_insn "*sse2_cvttpd2dq_mask_1"
6406 [(set (match_operand:V4SI 0 "register_operand" "=v")
6407 (vec_concat:V4SI
6408 (vec_merge:V2SI
6409 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6410 (const_vector:V2SI [(const_int 0) (const_int 0)])
6411 (match_operand:QI 2 "register_operand" "Yk"))
6412 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
6413 "TARGET_AVX512VL"
6414 "vcvttpd2dq{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
6415 [(set_attr "type" "ssecvt")
6416 (set_attr "prefix" "evex")
5608 (set_attr "mode" "TI")]) 6417 (set_attr "mode" "TI")])
5609 6418
5610 (define_insn "sse2_cvtsd2ss<round_name>" 6419 (define_insn "sse2_cvtsd2ss<round_name>"
5611 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v") 6420 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
5612 (vec_merge:V4SF 6421 (vec_merge:V4SF
5724 "TARGET_SSE2" 6533 "TARGET_SSE2"
5725 "operands[2] = CONST0_RTX (V2SFmode);") 6534 "operands[2] = CONST0_RTX (V2SFmode);")
5726 6535
5727 (define_expand "sse2_cvtpd2ps_mask" 6536 (define_expand "sse2_cvtpd2ps_mask"
5728 [(set (match_operand:V4SF 0 "register_operand") 6537 [(set (match_operand:V4SF 0 "register_operand")
5729 (vec_merge:V4SF 6538 (vec_concat:V4SF
5730 (vec_concat:V4SF 6539 (vec_merge:V2SF
5731 (float_truncate:V2SF 6540 (float_truncate:V2SF
5732 (match_operand:V2DF 1 "vector_operand")) 6541 (match_operand:V2DF 1 "vector_operand"))
5733 (match_dup 4)) 6542 (vec_select:V2SF
5734 (match_operand:V4SF 2 "register_operand") 6543 (match_operand:V4SF 2 "nonimm_or_0_operand")
5735 (match_operand:QI 3 "register_operand")))] 6544 (parallel [(const_int 0) (const_int 1)]))
6545 (match_operand:QI 3 "register_operand"))
6546 (match_dup 4)))]
5736 "TARGET_SSE2" 6547 "TARGET_SSE2"
5737 "operands[4] = CONST0_RTX (V2SFmode);") 6548 "operands[4] = CONST0_RTX (V2SFmode);")
5738 6549
5739 (define_insn "*sse2_cvtpd2ps<mask_name>" 6550 (define_insn "*sse2_cvtpd2ps"
5740 [(set (match_operand:V4SF 0 "register_operand" "=v") 6551 [(set (match_operand:V4SF 0 "register_operand" "=v")
5741 (vec_concat:V4SF 6552 (vec_concat:V4SF
5742 (float_truncate:V2SF 6553 (float_truncate:V2SF
5743 (match_operand:V2DF 1 "vector_operand" "vBm")) 6554 (match_operand:V2DF 1 "vector_operand" "vBm"))
5744 (match_operand:V2SF 2 "const0_operand")))] 6555 (match_operand:V2SF 2 "const0_operand" "C")))]
5745 "TARGET_SSE2 && <mask_avx512vl_condition>" 6556 "TARGET_SSE2"
5746 { 6557 {
5747 if (TARGET_AVX) 6558 if (TARGET_AVX)
5748 return "vcvtpd2ps{x}\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}"; 6559 return "vcvtpd2ps{x}\t{%1, %0|%0, %1}";
5749 else 6560 else
5750 return "cvtpd2ps\t{%1, %0|%0, %1}"; 6561 return "cvtpd2ps\t{%1, %0|%0, %1}";
5751 } 6562 }
5752 [(set_attr "type" "ssecvt") 6563 [(set_attr "type" "ssecvt")
5753 (set_attr "amdfam10_decode" "double") 6564 (set_attr "amdfam10_decode" "double")
5754 (set_attr "athlon_decode" "vector") 6565 (set_attr "athlon_decode" "vector")
5755 (set_attr "bdver1_decode" "double") 6566 (set_attr "bdver1_decode" "double")
5756 (set_attr "prefix_data16" "1") 6567 (set_attr "prefix_data16" "1")
5757 (set_attr "prefix" "maybe_vex") 6568 (set_attr "prefix" "maybe_vex")
6569 (set_attr "mode" "V4SF")])
6570
6571 (define_insn "*sse2_cvtpd2ps_mask"
6572 [(set (match_operand:V4SF 0 "register_operand" "=v")
6573 (vec_concat:V4SF
6574 (vec_merge:V2SF
6575 (float_truncate:V2SF
6576 (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6577 (vec_select:V2SF
6578 (match_operand:V4SF 2 "nonimm_or_0_operand" "0C")
6579 (parallel [(const_int 0) (const_int 1)]))
6580 (match_operand:QI 3 "register_operand" "Yk"))
6581 (match_operand:V2SF 4 "const0_operand" "C")))]
6582 "TARGET_AVX512VL"
6583 "vcvtpd2ps{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
6584 [(set_attr "type" "ssecvt")
6585 (set_attr "prefix" "evex")
6586 (set_attr "mode" "V4SF")])
6587
6588 (define_insn "*sse2_cvtpd2ps_mask_1"
6589 [(set (match_operand:V4SF 0 "register_operand" "=v")
6590 (vec_concat:V4SF
6591 (vec_merge:V2SF
6592 (float_truncate:V2SF
6593 (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
6594 (match_operand:V2SF 3 "const0_operand" "C")
6595 (match_operand:QI 2 "register_operand" "Yk"))
6596 (match_operand:V2SF 4 "const0_operand" "C")))]
6597 "TARGET_AVX512VL"
6598 "vcvtpd2ps{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
6599 [(set_attr "type" "ssecvt")
6600 (set_attr "prefix" "evex")
5758 (set_attr "mode" "V4SF")]) 6601 (set_attr "mode" "V4SF")])
5759 6602
5760 ;; For <sse2_avx_avx512f>_cvtps2pd<avxsizesuffix> insn pattern 6603 ;; For <sse2_avx_avx512f>_cvtps2pd<avxsizesuffix> insn pattern
5761 (define_mode_attr sf2dfmode 6604 (define_mode_attr sf2dfmode
5762 [(V8DF "V8SF") (V4DF "V4SF")]) 6605 [(V8DF "V8SF") (V4DF "V4SF")])
5798 [(set_attr "type" "ssecvt") 6641 [(set_attr "type" "ssecvt")
5799 (set_attr "prefix" "evex") 6642 (set_attr "prefix" "evex")
5800 (set_attr "mode" "V8DF")]) 6643 (set_attr "mode" "V8DF")])
5801 6644
5802 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>" 6645 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
5803 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 6646 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
5804 (unspec:<avx512fmaskmode> 6647 (unspec:<avx512fmaskmode>
5805 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")] 6648 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")]
5806 UNSPEC_CVTINT2MASK))] 6649 UNSPEC_CVTINT2MASK))]
5807 "TARGET_AVX512BW" 6650 "TARGET_AVX512BW"
5808 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}" 6651 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
5809 [(set_attr "prefix" "evex") 6652 [(set_attr "prefix" "evex")
5810 (set_attr "mode" "<sseinsnmode>")]) 6653 (set_attr "mode" "<sseinsnmode>")])
5811 6654
5812 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>" 6655 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
5813 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 6656 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
5814 (unspec:<avx512fmaskmode> 6657 (unspec:<avx512fmaskmode>
5815 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")] 6658 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")]
5816 UNSPEC_CVTINT2MASK))] 6659 UNSPEC_CVTINT2MASK))]
5817 "TARGET_AVX512DQ" 6660 "TARGET_AVX512DQ"
5818 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}" 6661 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
5834 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>" 6677 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5835 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v") 6678 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
5836 (vec_merge:VI12_AVX512VL 6679 (vec_merge:VI12_AVX512VL
5837 (match_operand:VI12_AVX512VL 2 "vector_all_ones_operand") 6680 (match_operand:VI12_AVX512VL 2 "vector_all_ones_operand")
5838 (match_operand:VI12_AVX512VL 3 "const0_operand") 6681 (match_operand:VI12_AVX512VL 3 "const0_operand")
5839 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))] 6682 (match_operand:<avx512fmaskmode> 1 "register_operand" "k")))]
5840 "TARGET_AVX512BW" 6683 "TARGET_AVX512BW"
5841 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}" 6684 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}"
5842 [(set_attr "prefix" "evex") 6685 [(set_attr "prefix" "evex")
5843 (set_attr "mode" "<sseinsnmode>")]) 6686 (set_attr "mode" "<sseinsnmode>")])
5844 6687
5846 [(set (match_operand:VI48_AVX512VL 0 "register_operand") 6689 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
5847 (vec_merge:VI48_AVX512VL 6690 (vec_merge:VI48_AVX512VL
5848 (match_dup 2) 6691 (match_dup 2)
5849 (match_dup 3) 6692 (match_dup 3)
5850 (match_operand:<avx512fmaskmode> 1 "register_operand")))] 6693 (match_operand:<avx512fmaskmode> 1 "register_operand")))]
5851 "TARGET_AVX512DQ" 6694 "TARGET_AVX512F"
5852 "{ 6695 "{
5853 operands[2] = CONSTM1_RTX (<MODE>mode); 6696 operands[2] = CONSTM1_RTX (<MODE>mode);
5854 operands[3] = CONST0_RTX (<MODE>mode); 6697 operands[3] = CONST0_RTX (<MODE>mode);
5855 }") 6698 }")
5856 6699
5857 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>" 6700 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5858 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v") 6701 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v,v")
5859 (vec_merge:VI48_AVX512VL 6702 (vec_merge:VI48_AVX512VL
5860 (match_operand:VI48_AVX512VL 2 "vector_all_ones_operand") 6703 (match_operand:VI48_AVX512VL 2 "vector_all_ones_operand")
5861 (match_operand:VI48_AVX512VL 3 "const0_operand") 6704 (match_operand:VI48_AVX512VL 3 "const0_operand")
5862 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))] 6705 (match_operand:<avx512fmaskmode> 1 "register_operand" "k,Yk")))]
5863 "TARGET_AVX512DQ" 6706 "TARGET_AVX512F"
5864 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}" 6707 "@
5865 [(set_attr "prefix" "evex") 6708 vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}
6709 vpternlog<ssemodesuffix>\t{$0x81, %0, %0, %0%{%1%}%{z%}|%0%{%1%}%{z%}, %0, %0, 0x81}"
6710 [(set_attr "isa" "avx512dq,*")
6711 (set_attr "length_immediate" "0,1")
6712 (set_attr "prefix" "evex")
5866 (set_attr "mode" "<sseinsnmode>")]) 6713 (set_attr "mode" "<sseinsnmode>")])
5867 6714
5868 (define_insn "sse2_cvtps2pd<mask_name>" 6715 (define_insn "sse2_cvtps2pd<mask_name>"
5869 [(set (match_operand:V2DF 0 "register_operand" "=v") 6716 [(set (match_operand:V2DF 0 "register_operand" "=v")
5870 (float_extend:V2DF 6717 (float_extend:V2DF
7188 (const_string "mmxcvt") 8035 (const_string "mmxcvt")
7189 (eq_attr "alternative" "8") 8036 (eq_attr "alternative" "8")
7190 (const_string "mmxmov") 8037 (const_string "mmxmov")
7191 ] 8038 ]
7192 (const_string "sselog"))) 8039 (const_string "sselog")))
8040 (set (attr "mmx_isa")
8041 (if_then_else (eq_attr "alternative" "7,8")
8042 (const_string "native")
8043 (const_string "*")))
7193 (set (attr "prefix_data16") 8044 (set (attr "prefix_data16")
7194 (if_then_else (eq_attr "alternative" "3,4") 8045 (if_then_else (eq_attr "alternative" "3,4")
7195 (const_string "1") 8046 (const_string "1")
7196 (const_string "*"))) 8047 (const_string "*")))
7197 (set (attr "prefix_extra") 8048 (set (attr "prefix_extra")
7223 "@ 8074 "@
7224 unpcklps\t{%2, %0|%0, %2} 8075 unpcklps\t{%2, %0|%0, %2}
7225 movss\t{%1, %0|%0, %1} 8076 movss\t{%1, %0|%0, %1}
7226 punpckldq\t{%2, %0|%0, %2} 8077 punpckldq\t{%2, %0|%0, %2}
7227 movd\t{%1, %0|%0, %1}" 8078 movd\t{%1, %0|%0, %1}"
7228 [(set_attr "type" "sselog,ssemov,mmxcvt,mmxmov") 8079 [(set_attr "mmx_isa" "*,*,native,native")
8080 (set_attr "type" "sselog,ssemov,mmxcvt,mmxmov")
7229 (set_attr "mode" "V4SF,SF,DI,DI")]) 8081 (set_attr "mode" "V4SF,SF,DI,DI")])
7230 8082
7231 (define_insn "*vec_concatv4sf" 8083 (define_insn "*vec_concatv4sf"
7232 [(set (match_operand:V4SF 0 "register_operand" "=x,v,x,v") 8084 [(set (match_operand:V4SF 0 "register_operand" "=x,v,x,v")
7233 (vec_concat:V4SF 8085 (vec_concat:V4SF
7241 vmovhps\t{%2, %1, %0|%0, %1, %q2}" 8093 vmovhps\t{%2, %1, %0|%0, %1, %q2}"
7242 [(set_attr "isa" "noavx,avx,noavx,avx") 8094 [(set_attr "isa" "noavx,avx,noavx,avx")
7243 (set_attr "type" "ssemov") 8095 (set_attr "type" "ssemov")
7244 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex") 8096 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex")
7245 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF")]) 8097 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF")])
8098
8099 (define_insn "*vec_concatv4sf_0"
8100 [(set (match_operand:V4SF 0 "register_operand" "=v")
8101 (vec_concat:V4SF
8102 (match_operand:V2SF 1 "nonimmediate_operand" "xm")
8103 (match_operand:V2SF 2 "const0_operand" " C")))]
8104 "TARGET_SSE2"
8105 "%vmovq\t{%1, %0|%0, %1}"
8106 [(set_attr "type" "ssemov")
8107 (set_attr "prefix" "maybe_vex")
8108 (set_attr "mode" "DF")])
7246 8109
7247 ;; Avoid combining registers from different units in a single alternative, 8110 ;; Avoid combining registers from different units in a single alternative,
7248 ;; see comment above inline_secondary_memory_needed function in i386.c 8111 ;; see comment above inline_secondary_memory_needed function in i386.c
7249 (define_insn "vec_set<mode>_0" 8112 (define_insn "vec_set<mode>_0"
7250 [(set (match_operand:VI4F_128 0 "nonimmediate_operand" 8113 [(set (match_operand:VI4F_128 0 "nonimmediate_operand"
7353 ;; All of vinsertps, vmovss, vmovd clear also the higher bits. 8216 ;; All of vinsertps, vmovss, vmovd clear also the higher bits.
7354 (define_insn "vec_set<mode>_0" 8217 (define_insn "vec_set<mode>_0"
7355 [(set (match_operand:VI4F_256_512 0 "register_operand" "=v,v,v") 8218 [(set (match_operand:VI4F_256_512 0 "register_operand" "=v,v,v")
7356 (vec_merge:VI4F_256_512 8219 (vec_merge:VI4F_256_512
7357 (vec_duplicate:VI4F_256_512 8220 (vec_duplicate:VI4F_256_512
7358 (match_operand:<ssescalarmode> 2 "general_operand" "v,m,r")) 8221 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "v,m,r"))
7359 (match_operand:VI4F_256_512 1 "const0_operand" "C,C,C") 8222 (match_operand:VI4F_256_512 1 "const0_operand" "C,C,C")
7360 (const_int 1)))] 8223 (const_int 1)))]
7361 "TARGET_AVX" 8224 "TARGET_AVX"
7362 "@ 8225 "@
7363 vinsertps\t{$0xe, %2, %2, %x0|%x0, %2, %2, 0xe} 8226 vinsertps\t{$0xe, %2, %2, %x0|%x0, %2, %2, 0xe}
7417 (match_dup 0) 8280 (match_dup 0)
7418 (const_int 1)))] 8281 (const_int 1)))]
7419 "TARGET_SSE && reload_completed" 8282 "TARGET_SSE && reload_completed"
7420 [(set (match_dup 0) (match_dup 1))] 8283 [(set (match_dup 0) (match_dup 1))]
7421 "operands[0] = adjust_address (operands[0], <ssescalarmode>mode, 0);") 8284 "operands[0] = adjust_address (operands[0], <ssescalarmode>mode, 0);")
8285
8286 ;; Standard scalar operation patterns which preserve the rest of the
8287 ;; vector for combiner.
8288 (define_insn "vec_setv2df_0"
8289 [(set (match_operand:V2DF 0 "register_operand" "=x,v,x,v")
8290 (vec_merge:V2DF
8291 (vec_duplicate:V2DF
8292 (match_operand:DF 2 "nonimmediate_operand" " x,v,m,m"))
8293 (match_operand:V2DF 1 "register_operand" " 0,v,0,v")
8294 (const_int 1)))]
8295 "TARGET_SSE2"
8296 "@
8297 movsd\t{%2, %0|%0, %2}
8298 vmovsd\t{%2, %1, %0|%0, %1, %2}
8299 movlpd\t{%2, %0|%0, %2}
8300 vmovlpd\t{%2, %1, %0|%0, %1, %2}"
8301 [(set_attr "isa" "noavx,avx,noavx,avx")
8302 (set_attr "type" "ssemov")
8303 (set_attr "mode" "DF")])
7422 8304
7423 (define_expand "vec_set<mode>" 8305 (define_expand "vec_set<mode>"
7424 [(match_operand:V 0 "register_operand") 8306 [(match_operand:V 0 "register_operand")
7425 (match_operand:<ssescalarmode> 1 "register_operand") 8307 (match_operand:<ssescalarmode> 1 "register_operand")
7426 (match_operand 2 "const_int_operand")] 8308 (match_operand 2 "const_int_operand")]
7835 (set_attr "isa" "avx512dq,noavx512dq") 8717 (set_attr "isa" "avx512dq,noavx512dq")
7836 (set_attr "length_immediate" "1") 8718 (set_attr "length_immediate" "1")
7837 (set_attr "prefix" "evex") 8719 (set_attr "prefix" "evex")
7838 (set_attr "mode" "<sseinsnmode>")]) 8720 (set_attr "mode" "<sseinsnmode>")])
7839 8721
8722 (define_mode_iterator VI48F_256_DQ
8723 [V8SI V8SF (V4DI "TARGET_AVX512DQ") (V4DF "TARGET_AVX512DQ")])
8724
7840 (define_expand "avx512vl_vextractf128<mode>" 8725 (define_expand "avx512vl_vextractf128<mode>"
7841 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand") 8726 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7842 (match_operand:VI48F_256 1 "register_operand") 8727 (match_operand:VI48F_256_DQ 1 "register_operand")
7843 (match_operand:SI 2 "const_0_to_1_operand") 8728 (match_operand:SI 2 "const_0_to_1_operand")
7844 (match_operand:<ssehalfvecmode> 3 "nonimm_or_0_operand") 8729 (match_operand:<ssehalfvecmode> 3 "nonimm_or_0_operand")
7845 (match_operand:QI 4 "register_operand")] 8730 (match_operand:QI 4 "register_operand")]
7846 "TARGET_AVX512DQ && TARGET_AVX512VL" 8731 "TARGET_AVX512VL"
7847 { 8732 {
7848 rtx (*insn)(rtx, rtx, rtx, rtx); 8733 rtx (*insn)(rtx, rtx, rtx, rtx);
7849 rtx dest = operands[0]; 8734 rtx dest = operands[0];
7850 8735
7851 if (MEM_P (dest) 8736 if (MEM_P (dest)
7909 (parallel [(const_int 0) (const_int 1) 8794 (parallel [(const_int 0) (const_int 1)
7910 (const_int 2) (const_int 3) 8795 (const_int 2) (const_int 3)
7911 (const_int 4) (const_int 5) 8796 (const_int 4) (const_int 5)
7912 (const_int 6) (const_int 7)])))] 8797 (const_int 6) (const_int 7)])))]
7913 "TARGET_AVX512F 8798 "TARGET_AVX512F
7914 && <mask_mode512bit_condition> 8799 && <mask_avx512dq_condition>
7915 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))" 8800 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
7916 { 8801 {
7917 if (<mask_applied> 8802 if (<mask_applied>
7918 || (!TARGET_AVX512VL 8803 || (!TARGET_AVX512VL
7919 && !REG_P (operands[0]) 8804 && !REG_P (operands[0])
7920 && EXT_REX_SSE_REG_P (operands[1]))) 8805 && EXT_REX_SSE_REG_P (operands[1])))
7921 return "vextract<shuffletype>32x8\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}"; 8806 {
8807 if (TARGET_AVX512DQ)
8808 return "vextract<shuffletype>32x8\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
8809 else
8810 return "vextract<shuffletype>64x4\t{$0x0, %1, %0|%0, %1, 0x0}";
8811 }
7922 else 8812 else
7923 return "#"; 8813 return "#";
7924 } 8814 }
7925 [(set_attr "type" "sselog1") 8815 [(set_attr "type" "sselog1")
7926 (set_attr "prefix_extra" "1") 8816 (set_attr "prefix_extra" "1")
8026 (match_operand:VI4F_256 1 "<store_mask_predicate>" 8916 (match_operand:VI4F_256 1 "<store_mask_predicate>"
8027 "v,<store_mask_constraint>") 8917 "v,<store_mask_constraint>")
8028 (parallel [(const_int 0) (const_int 1) 8918 (parallel [(const_int 0) (const_int 1)
8029 (const_int 2) (const_int 3)])))] 8919 (const_int 2) (const_int 3)])))]
8030 "TARGET_AVX 8920 "TARGET_AVX
8031 && <mask_avx512vl_condition> && <mask_avx512dq_condition> 8921 && <mask_avx512vl_condition>
8032 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))" 8922 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
8033 { 8923 {
8034 if (<mask_applied>) 8924 if (<mask_applied>)
8035 return "vextract<shuffletype>32x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}"; 8925 return "vextract<shuffletype>32x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
8036 else 8926 else
8344 DONE; 9234 DONE;
8345 }) 9235 })
8346 9236
8347 (define_expand "vec_extract<mode><ssehalfvecmodelower>" 9237 (define_expand "vec_extract<mode><ssehalfvecmodelower>"
8348 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand") 9238 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
8349 (match_operand:V_512 1 "register_operand") 9239 (match_operand:V_256_512 1 "register_operand")
8350 (match_operand 2 "const_0_to_1_operand")] 9240 (match_operand 2 "const_0_to_1_operand")]
8351 "TARGET_AVX512F" 9241 "TARGET_AVX"
8352 { 9242 {
8353 if (INTVAL (operands[2])) 9243 if (INTVAL (operands[2]))
8354 emit_insn (gen_vec_extract_hi_<mode> (operands[0], operands[1])); 9244 emit_insn (gen_vec_extract_hi_<mode> (operands[0], operands[1]));
8355 else 9245 else
8356 emit_insn (gen_vec_extract_lo_<mode> (operands[0], operands[1])); 9246 emit_insn (gen_vec_extract_lo_<mode> (operands[0], operands[1]));
8874 [(set (match_operand:VF_128 0 "register_operand" "=v") 9764 [(set (match_operand:VF_128 0 "register_operand" "=v")
8875 (vec_merge:VF_128 9765 (vec_merge:VF_128
8876 (unspec:VF_128 9766 (unspec:VF_128
8877 [(match_operand:VF_128 1 "register_operand" "0") 9767 [(match_operand:VF_128 1 "register_operand" "0")
8878 (match_operand:VF_128 2 "register_operand" "v") 9768 (match_operand:VF_128 2 "register_operand" "v")
8879 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>") 9769 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
8880 (match_operand:SI 4 "const_0_to_255_operand")] 9770 (match_operand:SI 4 "const_0_to_255_operand")]
8881 UNSPEC_FIXUPIMM) 9771 UNSPEC_FIXUPIMM)
8882 (match_dup 1) 9772 (match_dup 1)
8883 (const_int 1)))] 9773 (const_int 1)))]
8884 "TARGET_AVX512F" 9774 "TARGET_AVX512F"
8891 (vec_merge:VF_128 9781 (vec_merge:VF_128
8892 (vec_merge:VF_128 9782 (vec_merge:VF_128
8893 (unspec:VF_128 9783 (unspec:VF_128
8894 [(match_operand:VF_128 1 "register_operand" "0") 9784 [(match_operand:VF_128 1 "register_operand" "0")
8895 (match_operand:VF_128 2 "register_operand" "v") 9785 (match_operand:VF_128 2 "register_operand" "v")
8896 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>") 9786 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
8897 (match_operand:SI 4 "const_0_to_255_operand")] 9787 (match_operand:SI 4 "const_0_to_255_operand")]
8898 UNSPEC_FIXUPIMM) 9788 UNSPEC_FIXUPIMM)
8899 (match_dup 1) 9789 (match_dup 1)
8900 (const_int 1)) 9790 (const_int 1))
8901 (match_dup 1) 9791 (match_dup 1)
8915 "vrndscale<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}" 9805 "vrndscale<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}"
8916 [(set_attr "length_immediate" "1") 9806 [(set_attr "length_immediate" "1")
8917 (set_attr "prefix" "evex") 9807 (set_attr "prefix" "evex")
8918 (set_attr "mode" "<MODE>")]) 9808 (set_attr "mode" "<MODE>")])
8919 9809
8920 (define_insn "avx512f_rndscale<mode><round_saeonly_name>" 9810 (define_insn "avx512f_rndscale<mode><mask_scalar_name><round_saeonly_scalar_name>"
8921 [(set (match_operand:VF_128 0 "register_operand" "=v") 9811 [(set (match_operand:VF_128 0 "register_operand" "=v")
8922 (vec_merge:VF_128 9812 (vec_merge:VF_128
8923 (unspec:VF_128 9813 (unspec:VF_128
8924 [(match_operand:VF_128 1 "register_operand" "v") 9814 [(match_operand:VF_128 2 "<round_saeonly_scalar_nimm_predicate>" "<round_saeonly_scalar_constraint>")
8925 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
8926 (match_operand:SI 3 "const_0_to_255_operand")] 9815 (match_operand:SI 3 "const_0_to_255_operand")]
8927 UNSPEC_ROUND) 9816 UNSPEC_ROUND)
8928 (match_dup 1) 9817 (match_operand:VF_128 1 "register_operand" "v")
8929 (const_int 1)))] 9818 (const_int 1)))]
8930 "TARGET_AVX512F" 9819 "TARGET_AVX512F"
8931 "vrndscale<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %<iptr>2<round_saeonly_op4>, %3}" 9820 "vrndscale<ssescalarmodesuffix>\t{%3, <round_saeonly_scalar_mask_op4>%2, %1, %0<mask_scalar_operand4>|%0<mask_scalar_operand4>, %1, %<iptr>2<round_saeonly_scalar_mask_op4>, %3}"
9821 [(set_attr "length_immediate" "1")
9822 (set_attr "prefix" "evex")
9823 (set_attr "mode" "<MODE>")])
9824
9825 (define_insn "*avx512f_rndscale<mode><round_saeonly_name>"
9826 [(set (match_operand:VF_128 0 "register_operand" "=v")
9827 (vec_merge:VF_128
9828 (vec_duplicate:VF_128
9829 (unspec:<ssescalarmode>
9830 [(match_operand:<ssescalarmode> 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
9831 (match_operand:SI 3 "const_0_to_255_operand")]
9832 UNSPEC_ROUND))
9833 (match_operand:VF_128 1 "register_operand" "v")
9834 (const_int 1)))]
9835 "TARGET_AVX512F"
9836 "vrndscale<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}"
8932 [(set_attr "length_immediate" "1") 9837 [(set_attr "length_immediate" "1")
8933 (set_attr "prefix" "evex") 9838 (set_attr "prefix" "evex")
8934 (set_attr "mode" "<MODE>")]) 9839 (set_attr "mode" "<MODE>")])
8935 9840
8936 ;; One bit in mask selects 2 elements. 9841 ;; One bit in mask selects 2 elements.
9299 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,o") 10204 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,o")
9300 (parallel [(const_int 1)])))] 10205 (parallel [(const_int 1)])))]
9301 "!TARGET_SSE2 && TARGET_SSE 10206 "!TARGET_SSE2 && TARGET_SSE
9302 && !(MEM_P (operands[0]) && MEM_P (operands[1]))" 10207 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
9303 "@ 10208 "@
9304 movhps\t{%1, %0|%q0, %1} 10209 movhps\t{%1, %0|%0, %1}
9305 movhlps\t{%1, %0|%0, %1} 10210 movhlps\t{%1, %0|%0, %1}
9306 movlps\t{%H1, %0|%0, %H1}" 10211 movlps\t{%H1, %0|%0, %H1}"
9307 [(set_attr "type" "ssemov") 10212 [(set_attr "type" "ssemov")
9308 (set_attr "mode" "V2SF,V4SF,V2SF")]) 10213 (set_attr "mode" "V2SF,V4SF,V2SF")])
9309 10214
9551 (set_attr "mode" "V2DF,DF,DF")]) 10456 (set_attr "mode" "V2DF,DF,DF")])
9552 10457
9553 (define_insn "vec_concatv2df" 10458 (define_insn "vec_concatv2df"
9554 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v,x,v,x,x, v,x,x") 10459 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v,x,v,x,x, v,x,x")
9555 (vec_concat:V2DF 10460 (vec_concat:V2DF
9556 (match_operand:DF 1 "nonimmediate_operand" " 0,x,v,m,m,0,x,xm,0,0") 10461 (match_operand:DF 1 "nonimmediate_operand" " 0,x,v,m,m,0,x,vm,0,0")
9557 (match_operand:DF 2 "nonimm_or_0_operand" " x,x,v,1,1,m,m, C,x,m")))] 10462 (match_operand:DF 2 "nonimm_or_0_operand" " x,x,v,1,1,m,m, C,x,m")))]
9558 "TARGET_SSE 10463 "TARGET_SSE
9559 && (!(MEM_P (operands[1]) && MEM_P (operands[2])) 10464 && (!(MEM_P (operands[1]) && MEM_P (operands[2]))
9560 || (TARGET_SSE3 && rtx_equal_p (operands[1], operands[2])))" 10465 || (TARGET_SSE3 && rtx_equal_p (operands[1], operands[2])))"
9561 "@ 10466 "@
9605 ;; vmovq clears also the higher bits. 10510 ;; vmovq clears also the higher bits.
9606 (define_insn "vec_set<mode>_0" 10511 (define_insn "vec_set<mode>_0"
9607 [(set (match_operand:VF2_512_256 0 "register_operand" "=v") 10512 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
9608 (vec_merge:VF2_512_256 10513 (vec_merge:VF2_512_256
9609 (vec_duplicate:VF2_512_256 10514 (vec_duplicate:VF2_512_256
9610 (match_operand:<ssescalarmode> 2 "general_operand" "xm")) 10515 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "xm"))
9611 (match_operand:VF2_512_256 1 "const0_operand" "C") 10516 (match_operand:VF2_512_256 1 "const0_operand" "C")
9612 (const_int 1)))] 10517 (const_int 1)))]
9613 "TARGET_AVX" 10518 "TARGET_AVX"
9614 "vmovq\t{%2, %x0|%x0, %2}" 10519 "vmovq\t{%2, %x0|%x0, %2}"
9615 [(set_attr "type" "ssemov") 10520 [(set_attr "type" "ssemov")
11121 (match_operand:VI12_AVX2 2 "vector_operand"))) 12026 (match_operand:VI12_AVX2 2 "vector_operand")))
11122 (match_dup 3)) 12027 (match_dup 3))
11123 (const_int 1))))] 12028 (const_int 1))))]
11124 "TARGET_SSE2" 12029 "TARGET_SSE2"
11125 { 12030 {
11126 operands[3] = CONST1_RTX(<MODE>mode); 12031 operands[3] = CONST1_RTX(<ssedoublemode>mode);
11127 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands); 12032 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
11128 }) 12033 })
11129 12034
11130 (define_expand "usadv16qi" 12035 (define_expand "usadv16qi"
11131 [(match_operand:V4SI 0 "register_operand") 12036 [(match_operand:V4SI 0 "register_operand")
11265 (const_string "1") 12170 (const_string "1")
11266 (const_string "0"))) 12171 (const_string "0")))
11267 (set_attr "mode" "<sseinsnmode>")]) 12172 (set_attr "mode" "<sseinsnmode>")])
11268 12173
11269 12174
12175 (define_expand "vec_shl_<mode>"
12176 [(set (match_dup 3)
12177 (ashift:V1TI
12178 (match_operand:V_128 1 "register_operand")
12179 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
12180 (set (match_operand:V_128 0 "register_operand") (match_dup 4))]
12181 "TARGET_SSE2"
12182 {
12183 operands[1] = gen_lowpart (V1TImode, operands[1]);
12184 operands[3] = gen_reg_rtx (V1TImode);
12185 operands[4] = gen_lowpart (<MODE>mode, operands[3]);
12186 })
12187
11270 (define_expand "vec_shr_<mode>" 12188 (define_expand "vec_shr_<mode>"
11271 [(set (match_dup 3) 12189 [(set (match_dup 3)
11272 (lshiftrt:V1TI 12190 (lshiftrt:V1TI
11273 (match_operand:VI_128 1 "register_operand") 12191 (match_operand:V_128 1 "register_operand")
11274 (match_operand:SI 2 "const_0_to_255_mul_8_operand"))) 12192 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
11275 (set (match_operand:VI_128 0 "register_operand") (match_dup 4))] 12193 (set (match_operand:V_128 0 "register_operand") (match_dup 4))]
11276 "TARGET_SSE2" 12194 "TARGET_SSE2"
11277 { 12195 {
11278 operands[1] = gen_lowpart (V1TImode, operands[1]); 12196 operands[1] = gen_lowpart (V1TImode, operands[1]);
11279 operands[3] = gen_reg_rtx (V1TImode); 12197 operands[3] = gen_reg_rtx (V1TImode);
11280 operands[4] = gen_lowpart (<MODE>mode, operands[3]); 12198 operands[4] = gen_lowpart (<MODE>mode, operands[3]);
11640 UNSPEC_MASKED_EQ))] 12558 UNSPEC_MASKED_EQ))]
11641 "TARGET_AVX512F" 12559 "TARGET_AVX512F"
11642 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);") 12560 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
11643 12561
11644 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1" 12562 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
11645 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk,Yk") 12563 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k,k")
11646 (unspec:<avx512fmaskmode> 12564 (unspec:<avx512fmaskmode>
11647 [(match_operand:VI12_AVX512VL 1 "nonimm_or_0_operand" "%v,v") 12565 [(match_operand:VI12_AVX512VL 1 "nonimm_or_0_operand" "%v,v")
11648 (match_operand:VI12_AVX512VL 2 "nonimm_or_0_operand" "vm,C")] 12566 (match_operand:VI12_AVX512VL 2 "nonimm_or_0_operand" "vm,C")]
11649 UNSPEC_MASKED_EQ))] 12567 UNSPEC_MASKED_EQ))]
11650 "TARGET_AVX512BW && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 12568 "TARGET_AVX512BW && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11655 (set_attr "prefix_extra" "1") 12573 (set_attr "prefix_extra" "1")
11656 (set_attr "prefix" "evex") 12574 (set_attr "prefix" "evex")
11657 (set_attr "mode" "<sseinsnmode>")]) 12575 (set_attr "mode" "<sseinsnmode>")])
11658 12576
11659 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1" 12577 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
11660 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk,Yk") 12578 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k,k")
11661 (unspec:<avx512fmaskmode> 12579 (unspec:<avx512fmaskmode>
11662 [(match_operand:VI48_AVX512VL 1 "nonimm_or_0_operand" "%v,v") 12580 [(match_operand:VI48_AVX512VL 1 "nonimm_or_0_operand" "%v,v")
11663 (match_operand:VI48_AVX512VL 2 "nonimm_or_0_operand" "vm,C")] 12581 (match_operand:VI48_AVX512VL 2 "nonimm_or_0_operand" "vm,C")]
11664 UNSPEC_MASKED_EQ))] 12582 UNSPEC_MASKED_EQ))]
11665 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 12583 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11746 (set_attr "prefix_extra" "1") 12664 (set_attr "prefix_extra" "1")
11747 (set_attr "prefix" "vex") 12665 (set_attr "prefix" "vex")
11748 (set_attr "mode" "OI")]) 12666 (set_attr "mode" "OI")])
11749 12667
11750 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>" 12668 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
11751 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 12669 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
11752 (unspec:<avx512fmaskmode> 12670 (unspec:<avx512fmaskmode>
11753 [(match_operand:VI48_AVX512VL 1 "register_operand" "v") 12671 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11754 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))] 12672 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
11755 "TARGET_AVX512F" 12673 "TARGET_AVX512F"
11756 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}" 12674 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11758 (set_attr "prefix_extra" "1") 12676 (set_attr "prefix_extra" "1")
11759 (set_attr "prefix" "evex") 12677 (set_attr "prefix" "evex")
11760 (set_attr "mode" "<sseinsnmode>")]) 12678 (set_attr "mode" "<sseinsnmode>")])
11761 12679
11762 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>" 12680 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
11763 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 12681 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
11764 (unspec:<avx512fmaskmode> 12682 (unspec:<avx512fmaskmode>
11765 [(match_operand:VI12_AVX512VL 1 "register_operand" "v") 12683 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11766 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))] 12684 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
11767 "TARGET_AVX512BW" 12685 "TARGET_AVX512BW"
11768 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}" 12686 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11963 [(set (match_operand:VI 0 "register_operand") 12881 [(set (match_operand:VI 0 "register_operand")
11964 (xor:VI (match_operand:VI 1 "vector_operand") 12882 (xor:VI (match_operand:VI 1 "vector_operand")
11965 (match_dup 2)))] 12883 (match_dup 2)))]
11966 "TARGET_SSE" 12884 "TARGET_SSE"
11967 { 12885 {
11968 operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode)); 12886 if (!TARGET_AVX512F)
11969 }) 12887 operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));
12888 else
12889 operands[2] = CONSTM1_RTX (<MODE>mode);
12890 })
12891
12892 (define_insn "<mask_codefor>one_cmpl<mode>2<mask_name>"
12893 [(set (match_operand:VI 0 "register_operand" "=v")
12894 (xor:VI (match_operand:VI 1 "nonimmediate_operand" "vm")
12895 (match_operand:VI 2 "vector_all_ones_operand" "BC")))]
12896 "TARGET_AVX512F"
12897 "vpternlog<ternlogsuffix>\t{$0x55, %1, %0, %0<mask_operand3>|%0<mask_operand3>, %0, %1, 0x55}"
12898 [(set_attr "type" "sselog")
12899 (set_attr "prefix" "evex")
12900 (set_attr "mode" "<sseinsnmode>")])
11970 12901
11971 (define_expand "<sse2_avx2>_andnot<mode>3" 12902 (define_expand "<sse2_avx2>_andnot<mode>3"
11972 [(set (match_operand:VI_AVX2 0 "register_operand") 12903 [(set (match_operand:VI_AVX2 0 "register_operand")
11973 (and:VI_AVX2 12904 (and:VI_AVX2
11974 (not:VI_AVX2 (match_operand:VI_AVX2 1 "register_operand")) 12905 (not:VI_AVX2 (match_operand:VI_AVX2 1 "register_operand"))
12002 (and:VI 12933 (and:VI
12003 (not:VI (match_operand:VI 1 "register_operand" "0,x,v")) 12934 (not:VI (match_operand:VI 1 "register_operand" "0,x,v"))
12004 (match_operand:VI 2 "vector_operand" "xBm,xm,vm")))] 12935 (match_operand:VI 2 "vector_operand" "xBm,xm,vm")))]
12005 "TARGET_SSE" 12936 "TARGET_SSE"
12006 { 12937 {
12007 static char buf[64]; 12938 char buf[64];
12008 const char *ops; 12939 const char *ops;
12009 const char *tmp; 12940 const char *tmp;
12010 const char *ssesuffix; 12941 const char *ssesuffix;
12011 12942
12012 switch (get_attr_mode (insn)) 12943 switch (get_attr_mode (insn))
12072 default: 13003 default:
12073 gcc_unreachable (); 13004 gcc_unreachable ();
12074 } 13005 }
12075 13006
12076 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix); 13007 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
12077 return buf; 13008 output_asm_insn (buf, operands);
13009 return "";
12078 } 13010 }
12079 [(set_attr "isa" "noavx,avx,avx") 13011 [(set_attr "isa" "noavx,avx,avx")
12080 (set_attr "type" "sselog") 13012 (set_attr "type" "sselog")
12081 (set (attr "prefix_data16") 13013 (set (attr "prefix_data16")
12082 (if_then_else 13014 (if_then_else
12084 (eq_attr "mode" "TI")) 13016 (eq_attr "mode" "TI"))
12085 (const_string "1") 13017 (const_string "1")
12086 (const_string "*"))) 13018 (const_string "*")))
12087 (set_attr "prefix" "orig,vex,evex") 13019 (set_attr "prefix" "orig,vex,evex")
12088 (set (attr "mode") 13020 (set (attr "mode")
12089 (cond [(and (match_test "<MODE_SIZE> == 16") 13021 (cond [(match_test "TARGET_AVX2")
12090 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
12091 (const_string "<ssePSmode>")
12092 (match_test "TARGET_AVX2")
12093 (const_string "<sseinsnmode>") 13022 (const_string "<sseinsnmode>")
12094 (match_test "TARGET_AVX") 13023 (match_test "TARGET_AVX")
12095 (if_then_else 13024 (if_then_else
12096 (match_test "<MODE_SIZE> > 16") 13025 (match_test "<MODE_SIZE> > 16")
12097 (const_string "V8SF") 13026 (const_string "V8SF")
12101 (const_string "V4SF") 13030 (const_string "V4SF")
12102 ] 13031 ]
12103 (const_string "<sseinsnmode>")))]) 13032 (const_string "<sseinsnmode>")))])
12104 13033
12105 (define_insn "*andnot<mode>3_bcst" 13034 (define_insn "*andnot<mode>3_bcst"
12106 [(set (match_operand:VI 0 "register_operand" "=v") 13035 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
12107 (and:VI 13036 (and:VI48_AVX512VL
12108 (not:VI48_AVX512VL 13037 (not:VI48_AVX512VL
12109 (match_operand:VI48_AVX512VL 1 "register_operand" "v")) 13038 (match_operand:VI48_AVX512VL 1 "register_operand" "v"))
12110 (vec_duplicate:VI48_AVX512VL 13039 (vec_duplicate:VI48_AVX512VL
12111 (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))] 13040 (match_operand:<ssescalarmode> 2 "memory_operand" "m"))))]
12112 "TARGET_AVX512F" 13041 "TARGET_AVX512F"
12147 (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,x,v") 13076 (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,x,v")
12148 (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))] 13077 (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
12149 "TARGET_SSE && <mask_mode512bit_condition> 13078 "TARGET_SSE && <mask_mode512bit_condition>
12150 && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 13079 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
12151 { 13080 {
12152 static char buf[64]; 13081 char buf[64];
12153 const char *ops; 13082 const char *ops;
12154 const char *tmp; 13083 const char *tmp;
12155 const char *ssesuffix; 13084 const char *ssesuffix;
12156 13085
12157 switch (get_attr_mode (insn)) 13086 switch (get_attr_mode (insn))
12212 default: 13141 default:
12213 gcc_unreachable (); 13142 gcc_unreachable ();
12214 } 13143 }
12215 13144
12216 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix); 13145 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
12217 return buf; 13146 output_asm_insn (buf, operands);
13147 return "";
12218 } 13148 }
12219 [(set_attr "isa" "noavx,avx,avx") 13149 [(set_attr "isa" "noavx,avx,avx")
12220 (set_attr "type" "sselog") 13150 (set_attr "type" "sselog")
12221 (set (attr "prefix_data16") 13151 (set (attr "prefix_data16")
12222 (if_then_else 13152 (if_then_else
12224 (eq_attr "mode" "TI")) 13154 (eq_attr "mode" "TI"))
12225 (const_string "1") 13155 (const_string "1")
12226 (const_string "*"))) 13156 (const_string "*")))
12227 (set_attr "prefix" "<mask_prefix3>,evex") 13157 (set_attr "prefix" "<mask_prefix3>,evex")
12228 (set (attr "mode") 13158 (set (attr "mode")
12229 (cond [(and (match_test "<MODE_SIZE> == 16") 13159 (cond [(match_test "TARGET_AVX2")
12230 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
12231 (const_string "<ssePSmode>")
12232 (match_test "TARGET_AVX2")
12233 (const_string "<sseinsnmode>") 13160 (const_string "<sseinsnmode>")
12234 (match_test "TARGET_AVX") 13161 (match_test "TARGET_AVX")
12235 (if_then_else 13162 (if_then_else
12236 (match_test "<MODE_SIZE> > 16") 13163 (match_test "<MODE_SIZE> > 16")
12237 (const_string "V8SF") 13164 (const_string "V8SF")
12247 (any_logic:VI12_AVX_AVX512F 13174 (any_logic:VI12_AVX_AVX512F
12248 (match_operand:VI12_AVX_AVX512F 1 "vector_operand" "%0,x,v") 13175 (match_operand:VI12_AVX_AVX512F 1 "vector_operand" "%0,x,v")
12249 (match_operand:VI12_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))] 13176 (match_operand:VI12_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
12250 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 13177 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
12251 { 13178 {
12252 static char buf[64]; 13179 char buf[64];
12253 const char *ops; 13180 const char *ops;
12254 const char *tmp; 13181 const char *tmp;
12255 const char *ssesuffix; 13182 const char *ssesuffix;
12256 13183
12257 switch (get_attr_mode (insn)) 13184 switch (get_attr_mode (insn))
12307 default: 13234 default:
12308 gcc_unreachable (); 13235 gcc_unreachable ();
12309 } 13236 }
12310 13237
12311 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix); 13238 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
12312 return buf; 13239 output_asm_insn (buf, operands);
13240 return "";
12313 } 13241 }
12314 [(set_attr "isa" "noavx,avx,avx") 13242 [(set_attr "isa" "noavx,avx,avx")
12315 (set_attr "type" "sselog") 13243 (set_attr "type" "sselog")
12316 (set (attr "prefix_data16") 13244 (set (attr "prefix_data16")
12317 (if_then_else 13245 (if_then_else
12319 (eq_attr "mode" "TI")) 13247 (eq_attr "mode" "TI"))
12320 (const_string "1") 13248 (const_string "1")
12321 (const_string "*"))) 13249 (const_string "*")))
12322 (set_attr "prefix" "orig,vex,evex") 13250 (set_attr "prefix" "orig,vex,evex")
12323 (set (attr "mode") 13251 (set (attr "mode")
12324 (cond [(and (match_test "<MODE_SIZE> == 16") 13252 (cond [(match_test "TARGET_AVX2")
12325 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
12326 (const_string "<ssePSmode>")
12327 (match_test "TARGET_AVX2")
12328 (const_string "<sseinsnmode>") 13253 (const_string "<sseinsnmode>")
12329 (match_test "TARGET_AVX") 13254 (match_test "TARGET_AVX")
12330 (if_then_else 13255 (if_then_else
12331 (match_test "<MODE_SIZE> > 16") 13256 (match_test "<MODE_SIZE> > 16")
12332 (const_string "V8SF") 13257 (const_string "V8SF")
12347 "vp<logic><ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}" 13272 "vp<logic><ssemodesuffix>\t{%1<avx512bcst>, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1<avx512bcst>}"
12348 [(set_attr "type" "sseiadd") 13273 [(set_attr "type" "sseiadd")
12349 (set_attr "prefix" "evex") 13274 (set_attr "prefix" "evex")
12350 (set_attr "mode" "<sseinsnmode>")]) 13275 (set_attr "mode" "<sseinsnmode>")])
12351 13276
13277 (define_mode_iterator VI1248_AVX512VLBW
13278 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX512VL && TARGET_AVX512BW")
13279 (V16QI "TARGET_AVX512VL && TARGET_AVX512BW")
13280 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512VL && TARGET_AVX512BW")
13281 (V8HI "TARGET_AVX512VL && TARGET_AVX512BW")
13282 V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
13283 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
13284
13285 (define_mode_iterator AVX512ZEXTMASK
13286 [(DI "TARGET_AVX512BW") (SI "TARGET_AVX512BW") HI])
13287
12352 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>" 13288 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
12353 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 13289 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
12354 (unspec:<avx512fmaskmode> 13290 (unspec:<avx512fmaskmode>
12355 [(match_operand:VI12_AVX512VL 1 "register_operand" "v") 13291 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
12356 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] 13292 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
12357 UNSPEC_TESTM))]
12358 "TARGET_AVX512BW"
12359 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
12360 [(set_attr "prefix" "evex")
12361 (set_attr "mode" "<sseinsnmode>")])
12362
12363 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
12364 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
12365 (unspec:<avx512fmaskmode>
12366 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
12367 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
12368 UNSPEC_TESTM))] 13293 UNSPEC_TESTM))]
12369 "TARGET_AVX512F" 13294 "TARGET_AVX512F"
12370 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}" 13295 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
12371 [(set_attr "prefix" "evex") 13296 [(set_attr "prefix" "evex")
12372 (set_attr "mode" "<sseinsnmode>")]) 13297 (set_attr "mode" "<sseinsnmode>")])
12373 13298
12374 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>" 13299 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
12375 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 13300 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
12376 (unspec:<avx512fmaskmode> 13301 (unspec:<avx512fmaskmode>
12377 [(match_operand:VI12_AVX512VL 1 "register_operand" "v") 13302 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
12378 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] 13303 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
12379 UNSPEC_TESTNM))]
12380 "TARGET_AVX512BW"
12381 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
12382 [(set_attr "prefix" "evex")
12383 (set_attr "mode" "<sseinsnmode>")])
12384
12385 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
12386 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
12387 (unspec:<avx512fmaskmode>
12388 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
12389 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
12390 UNSPEC_TESTNM))] 13304 UNSPEC_TESTNM))]
12391 "TARGET_AVX512F" 13305 "TARGET_AVX512F"
12392 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}" 13306 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
12393 [(set_attr "prefix" "evex") 13307 [(set_attr "prefix" "evex")
12394 (set_attr "mode" "<sseinsnmode>")]) 13308 (set_attr "mode" "<sseinsnmode>")])
12395 13309
13310 (define_insn "*<avx512>_testm<VI1248_AVX512VLBW:mode>3_zext"
13311 [(set (match_operand:AVX512ZEXTMASK 0 "register_operand" "=k")
13312 (zero_extend:AVX512ZEXTMASK
13313 (unspec:<VI1248_AVX512VLBW:avx512fmaskmode>
13314 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
13315 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
13316 UNSPEC_TESTM)))]
13317 "TARGET_AVX512BW
13318 && (<AVX512ZEXTMASK:MODE_SIZE>
13319 > GET_MODE_SIZE (<VI1248_AVX512VLBW:avx512fmaskmode>mode))"
13320 "vptestm<VI1248_AVX512VLBW:ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
13321 [(set_attr "prefix" "evex")
13322 (set_attr "mode" "<VI1248_AVX512VLBW:sseinsnmode>")])
13323
13324 (define_insn "*<avx512>_testm<VI1248_AVX512VLBW:mode>3_zext_mask"
13325 [(set (match_operand:AVX512ZEXTMASK 0 "register_operand" "=k")
13326 (zero_extend:AVX512ZEXTMASK
13327 (and:<VI1248_AVX512VLBW:avx512fmaskmode>
13328 (unspec:<VI1248_AVX512VLBW:avx512fmaskmode>
13329 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
13330 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
13331 UNSPEC_TESTM)
13332 (match_operand:<VI1248_AVX512VLBW:avx512fmaskmode> 3 "register_operand" "Yk"))))]
13333 "TARGET_AVX512BW
13334 && (<AVX512ZEXTMASK:MODE_SIZE>
13335 > GET_MODE_SIZE (<VI1248_AVX512VLBW:avx512fmaskmode>mode))"
13336 "vptestm<VI1248_AVX512VLBW:ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
13337 [(set_attr "prefix" "evex")
13338 (set_attr "mode" "<VI1248_AVX512VLBW:sseinsnmode>")])
13339
13340 (define_insn "*<avx512>_testnm<VI1248_AVX512VLBW:mode>3_zext"
13341 [(set (match_operand:AVX512ZEXTMASK 0 "register_operand" "=k")
13342 (zero_extend:AVX512ZEXTMASK
13343 (unspec:<VI1248_AVX512VLBW:avx512fmaskmode>
13344 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
13345 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
13346 UNSPEC_TESTNM)))]
13347 "TARGET_AVX512BW
13348 && (<AVX512ZEXTMASK:MODE_SIZE>
13349 > GET_MODE_SIZE (<VI1248_AVX512VLBW:avx512fmaskmode>mode))"
13350 "vptestnm<VI1248_AVX512VLBW:ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
13351 [(set_attr "prefix" "evex")
13352 (set_attr "mode" "<VI1248_AVX512VLBW:sseinsnmode>")])
13353
13354 (define_insn "*<avx512>_testnm<VI1248_AVX512VLBW:mode>3_zext_mask"
13355 [(set (match_operand:AVX512ZEXTMASK 0 "register_operand" "=k")
13356 (zero_extend:AVX512ZEXTMASK
13357 (and:<VI1248_AVX512VLBW:avx512fmaskmode>
13358 (unspec:<VI1248_AVX512VLBW:avx512fmaskmode>
13359 [(match_operand:VI1248_AVX512VLBW 1 "register_operand" "v")
13360 (match_operand:VI1248_AVX512VLBW 2 "nonimmediate_operand" "vm")]
13361 UNSPEC_TESTNM)
13362 (match_operand:<VI1248_AVX512VLBW:avx512fmaskmode> 3 "register_operand" "Yk"))))]
13363 "TARGET_AVX512BW
13364 && (<AVX512ZEXTMASK:MODE_SIZE>
13365 > GET_MODE_SIZE (<VI1248_AVX512VLBW:avx512fmaskmode>mode))"
13366 "vptestnm<VI1248_AVX512VLBW:ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
13367 [(set_attr "prefix" "evex")
13368 (set_attr "mode" "<VI1248_AVX512VLBW:sseinsnmode>")])
13369
12396 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 13370 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12397 ;; 13371 ;;
12398 ;; Parallel integral element swizzling 13372 ;; Parallel integral element swizzling
12399 ;; 13373 ;;
12400 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 13374 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12410 ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0); 13384 ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
12411 DONE; 13385 DONE;
12412 }) 13386 })
12413 13387
12414 (define_expand "vec_pack_trunc_qi" 13388 (define_expand "vec_pack_trunc_qi"
12415 [(set (match_operand:HI 0 ("register_operand")) 13389 [(set (match_operand:HI 0 "register_operand")
12416 (ior:HI (ashift:HI (zero_extend:HI (match_operand:QI 2 ("register_operand"))) 13390 (ior:HI (ashift:HI (zero_extend:HI (match_operand:QI 2 "register_operand"))
12417 (const_int 8)) 13391 (const_int 8))
12418 (zero_extend:HI (match_operand:QI 1 ("register_operand")))))] 13392 (zero_extend:HI (match_operand:QI 1 "register_operand"))))]
12419 "TARGET_AVX512F") 13393 "TARGET_AVX512F")
12420 13394
12421 (define_expand "vec_pack_trunc_<mode>" 13395 (define_expand "vec_pack_trunc_<mode>"
12422 [(set (match_operand:<DOUBLEMASKMODE> 0 ("register_operand")) 13396 [(set (match_operand:<DOUBLEMASKMODE> 0 "register_operand")
12423 (ior:<DOUBLEMASKMODE> (ashift:<DOUBLEMASKMODE> (zero_extend:<DOUBLEMASKMODE> (match_operand:SWI24 2 ("register_operand"))) 13397 (ior:<DOUBLEMASKMODE>
12424 (match_dup 3)) 13398 (ashift:<DOUBLEMASKMODE>
12425 (zero_extend:<DOUBLEMASKMODE> (match_operand:SWI24 1 ("register_operand")))))] 13399 (zero_extend:<DOUBLEMASKMODE>
13400 (match_operand:SWI24 2 "register_operand"))
13401 (match_dup 3))
13402 (zero_extend:<DOUBLEMASKMODE>
13403 (match_operand:SWI24 1 "register_operand"))))]
12426 "TARGET_AVX512BW" 13404 "TARGET_AVX512BW"
12427 { 13405 {
12428 operands[3] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode)); 13406 operands[3] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));
13407 })
13408
13409 (define_expand "vec_pack_sbool_trunc_qi"
13410 [(match_operand:QI 0 "register_operand")
13411 (match_operand:QI 1 "register_operand")
13412 (match_operand:QI 2 "register_operand")
13413 (match_operand:QI 3 "const_int_operand")]
13414 "TARGET_AVX512F"
13415 {
13416 HOST_WIDE_INT nunits = INTVAL (operands[3]);
13417 rtx mask, tem1, tem2;
13418 if (nunits != 8 && nunits != 4)
13419 FAIL;
13420 mask = gen_reg_rtx (QImode);
13421 emit_move_insn (mask, GEN_INT ((1 << (nunits / 2)) - 1));
13422 tem1 = gen_reg_rtx (QImode);
13423 emit_insn (gen_kandqi (tem1, operands[1], mask));
13424 if (TARGET_AVX512DQ)
13425 {
13426 tem2 = gen_reg_rtx (QImode);
13427 emit_insn (gen_kashiftqi (tem2, operands[2],
13428 GEN_INT (nunits / 2)));
13429 }
13430 else
13431 {
13432 tem2 = gen_reg_rtx (HImode);
13433 emit_insn (gen_kashifthi (tem2, lowpart_subreg (HImode, operands[2],
13434 QImode),
13435 GEN_INT (nunits / 2)));
13436 tem2 = lowpart_subreg (QImode, tem2, HImode);
13437 }
13438 emit_insn (gen_kiorqi (operands[0], tem1, tem2));
13439 DONE;
12429 }) 13440 })
12430 13441
12431 (define_insn "<sse2_avx2>_packsswb<mask_name>" 13442 (define_insn "<sse2_avx2>_packsswb<mask_name>"
12432 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v") 13443 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v")
12433 (vec_concat:VI1_AVX512 13444 (vec_concat:VI1_AVX512
13061 if (which_alternative == 0) 14072 if (which_alternative == 0)
13062 return "vinsert<shuffletype><extract_suf>\t{$0, %2, %1, %0|%0, %1, %2, 0}"; 14073 return "vinsert<shuffletype><extract_suf>\t{$0, %2, %1, %0|%0, %1, %2, 0}";
13063 switch (<MODE>mode) 14074 switch (<MODE>mode)
13064 { 14075 {
13065 case E_V8DFmode: 14076 case E_V8DFmode:
13066 return "vmovapd\t{%2, %x0|%x0, %2}"; 14077 if (misaligned_operand (operands[2], <ssequartermode>mode))
14078 return "vmovupd\t{%2, %x0|%x0, %2}";
14079 else
14080 return "vmovapd\t{%2, %x0|%x0, %2}";
13067 case E_V16SFmode: 14081 case E_V16SFmode:
13068 return "vmovaps\t{%2, %x0|%x0, %2}"; 14082 if (misaligned_operand (operands[2], <ssequartermode>mode))
14083 return "vmovups\t{%2, %x0|%x0, %2}";
14084 else
14085 return "vmovaps\t{%2, %x0|%x0, %2}";
13069 case E_V8DImode: 14086 case E_V8DImode:
13070 return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" 14087 if (misaligned_operand (operands[2], <ssequartermode>mode))
13071 : "vmovdqa\t{%2, %x0|%x0, %2}"; 14088 return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}"
14089 : "vmovdqu\t{%2, %x0|%x0, %2}";
14090 else
14091 return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}"
14092 : "vmovdqa\t{%2, %x0|%x0, %2}";
13072 case E_V16SImode: 14093 case E_V16SImode:
13073 return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" 14094 if (misaligned_operand (operands[2], <ssequartermode>mode))
13074 : "vmovdqa\t{%2, %x0|%x0, %2}"; 14095 return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}"
14096 : "vmovdqu\t{%2, %x0|%x0, %2}";
14097 else
14098 return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}"
14099 : "vmovdqa\t{%2, %x0|%x0, %2}";
13075 default: 14100 default:
13076 gcc_unreachable (); 14101 gcc_unreachable ();
13077 } 14102 }
13078 } 14103 }
13079 [(set_attr "type" "sselog,ssemov,ssemov") 14104 [(set_attr "type" "sselog,ssemov,ssemov")
13222 [(set (match_operand:VI8F_256 0 "register_operand" "=v") 14247 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
13223 (vec_select:VI8F_256 14248 (vec_select:VI8F_256
13224 (vec_concat:<ssedoublemode> 14249 (vec_concat:<ssedoublemode>
13225 (match_operand:VI8F_256 1 "register_operand" "v") 14250 (match_operand:VI8F_256 1 "register_operand" "v")
13226 (match_operand:VI8F_256 2 "nonimmediate_operand" "vm")) 14251 (match_operand:VI8F_256 2 "nonimmediate_operand" "vm"))
13227 (parallel [(match_operand 3 "const_0_to_3_operand") 14252 (parallel [(match_operand 3 "const_0_to_3_operand")
13228 (match_operand 4 "const_0_to_3_operand") 14253 (match_operand 4 "const_0_to_3_operand")
13229 (match_operand 5 "const_4_to_7_operand") 14254 (match_operand 5 "const_4_to_7_operand")
13230 (match_operand 6 "const_4_to_7_operand")])))] 14255 (match_operand 6 "const_4_to_7_operand")])))]
13231 "TARGET_AVX512VL 14256 "TARGET_AVX512VL
13232 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1) 14257 && (INTVAL (operands[3]) & 1) == 0
13233 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1))" 14258 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
14259 && (INTVAL (operands[5]) & 1) == 0
14260 && INTVAL (operands[5]) == INTVAL (operands[6]) - 1"
13234 { 14261 {
13235 int mask; 14262 int mask;
13236 mask = INTVAL (operands[3]) / 2; 14263 mask = INTVAL (operands[3]) / 2;
13237 mask |= (INTVAL (operands[5]) - 4) / 2 << 1; 14264 mask |= (INTVAL (operands[5]) - 4) / 2 << 1;
13238 operands[3] = GEN_INT (mask); 14265 operands[3] = GEN_INT (mask);
13271 [(set (match_operand:V8FI 0 "register_operand" "=v") 14298 [(set (match_operand:V8FI 0 "register_operand" "=v")
13272 (vec_select:V8FI 14299 (vec_select:V8FI
13273 (vec_concat:<ssedoublemode> 14300 (vec_concat:<ssedoublemode>
13274 (match_operand:V8FI 1 "register_operand" "v") 14301 (match_operand:V8FI 1 "register_operand" "v")
13275 (match_operand:V8FI 2 "nonimmediate_operand" "vm")) 14302 (match_operand:V8FI 2 "nonimmediate_operand" "vm"))
13276 (parallel [(match_operand 3 "const_0_to_7_operand") 14303 (parallel [(match_operand 3 "const_0_to_7_operand")
13277 (match_operand 4 "const_0_to_7_operand") 14304 (match_operand 4 "const_0_to_7_operand")
13278 (match_operand 5 "const_0_to_7_operand") 14305 (match_operand 5 "const_0_to_7_operand")
13279 (match_operand 6 "const_0_to_7_operand") 14306 (match_operand 6 "const_0_to_7_operand")
13280 (match_operand 7 "const_8_to_15_operand") 14307 (match_operand 7 "const_8_to_15_operand")
13281 (match_operand 8 "const_8_to_15_operand") 14308 (match_operand 8 "const_8_to_15_operand")
13282 (match_operand 9 "const_8_to_15_operand") 14309 (match_operand 9 "const_8_to_15_operand")
13283 (match_operand 10 "const_8_to_15_operand")])))] 14310 (match_operand 10 "const_8_to_15_operand")])))]
13284 "TARGET_AVX512F 14311 "TARGET_AVX512F
13285 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1) 14312 && (INTVAL (operands[3]) & 1) == 0
13286 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1) 14313 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
13287 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1) 14314 && (INTVAL (operands[5]) & 1) == 0
13288 && INTVAL (operands[9]) == (INTVAL (operands[10]) - 1))" 14315 && INTVAL (operands[5]) == INTVAL (operands[6]) - 1
14316 && (INTVAL (operands[7]) & 1) == 0
14317 && INTVAL (operands[7]) == INTVAL (operands[8]) - 1
14318 && (INTVAL (operands[9]) & 1) == 0
14319 && INTVAL (operands[9]) == INTVAL (operands[10]) - 1"
13289 { 14320 {
13290 int mask; 14321 int mask;
13291 mask = INTVAL (operands[3]) / 2; 14322 mask = INTVAL (operands[3]) / 2;
13292 mask |= INTVAL (operands[5]) / 2 << 2; 14323 mask |= INTVAL (operands[5]) / 2 << 2;
13293 mask |= (INTVAL (operands[7]) - 8) / 2 << 4; 14324 mask |= (INTVAL (operands[7]) - 8) / 2 << 4;
13329 [(set (match_operand:VI4F_256 0 "register_operand" "=v") 14360 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
13330 (vec_select:VI4F_256 14361 (vec_select:VI4F_256
13331 (vec_concat:<ssedoublemode> 14362 (vec_concat:<ssedoublemode>
13332 (match_operand:VI4F_256 1 "register_operand" "v") 14363 (match_operand:VI4F_256 1 "register_operand" "v")
13333 (match_operand:VI4F_256 2 "nonimmediate_operand" "vm")) 14364 (match_operand:VI4F_256 2 "nonimmediate_operand" "vm"))
13334 (parallel [(match_operand 3 "const_0_to_7_operand") 14365 (parallel [(match_operand 3 "const_0_to_7_operand")
13335 (match_operand 4 "const_0_to_7_operand") 14366 (match_operand 4 "const_0_to_7_operand")
13336 (match_operand 5 "const_0_to_7_operand") 14367 (match_operand 5 "const_0_to_7_operand")
13337 (match_operand 6 "const_0_to_7_operand") 14368 (match_operand 6 "const_0_to_7_operand")
13338 (match_operand 7 "const_8_to_15_operand") 14369 (match_operand 7 "const_8_to_15_operand")
13339 (match_operand 8 "const_8_to_15_operand") 14370 (match_operand 8 "const_8_to_15_operand")
13340 (match_operand 9 "const_8_to_15_operand") 14371 (match_operand 9 "const_8_to_15_operand")
13341 (match_operand 10 "const_8_to_15_operand")])))] 14372 (match_operand 10 "const_8_to_15_operand")])))]
13342 "TARGET_AVX512VL 14373 "TARGET_AVX512VL
13343 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1) 14374 && (INTVAL (operands[3]) & 3) == 0
13344 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2) 14375 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
13345 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3) 14376 && INTVAL (operands[3]) == INTVAL (operands[5]) - 2
13346 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1) 14377 && INTVAL (operands[3]) == INTVAL (operands[6]) - 3
13347 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2) 14378 && (INTVAL (operands[7]) & 3) == 0
13348 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3))" 14379 && INTVAL (operands[7]) == INTVAL (operands[8]) - 1
14380 && INTVAL (operands[7]) == INTVAL (operands[9]) - 2
14381 && INTVAL (operands[7]) == INTVAL (operands[10]) - 3"
13349 { 14382 {
13350 int mask; 14383 int mask;
13351 mask = INTVAL (operands[3]) / 4; 14384 mask = INTVAL (operands[3]) / 4;
13352 mask |= (INTVAL (operands[7]) - 8) / 4 << 1; 14385 mask |= (INTVAL (operands[7]) - 8) / 4 << 1;
13353 operands[3] = GEN_INT (mask); 14386 operands[3] = GEN_INT (mask);
13395 [(set (match_operand:V16FI 0 "register_operand" "=v") 14428 [(set (match_operand:V16FI 0 "register_operand" "=v")
13396 (vec_select:V16FI 14429 (vec_select:V16FI
13397 (vec_concat:<ssedoublemode> 14430 (vec_concat:<ssedoublemode>
13398 (match_operand:V16FI 1 "register_operand" "v") 14431 (match_operand:V16FI 1 "register_operand" "v")
13399 (match_operand:V16FI 2 "nonimmediate_operand" "vm")) 14432 (match_operand:V16FI 2 "nonimmediate_operand" "vm"))
13400 (parallel [(match_operand 3 "const_0_to_15_operand") 14433 (parallel [(match_operand 3 "const_0_to_15_operand")
13401 (match_operand 4 "const_0_to_15_operand") 14434 (match_operand 4 "const_0_to_15_operand")
13402 (match_operand 5 "const_0_to_15_operand") 14435 (match_operand 5 "const_0_to_15_operand")
13403 (match_operand 6 "const_0_to_15_operand") 14436 (match_operand 6 "const_0_to_15_operand")
13404 (match_operand 7 "const_0_to_15_operand") 14437 (match_operand 7 "const_0_to_15_operand")
13405 (match_operand 8 "const_0_to_15_operand") 14438 (match_operand 8 "const_0_to_15_operand")
13406 (match_operand 9 "const_0_to_15_operand") 14439 (match_operand 9 "const_0_to_15_operand")
13407 (match_operand 10 "const_0_to_15_operand") 14440 (match_operand 10 "const_0_to_15_operand")
13408 (match_operand 11 "const_16_to_31_operand") 14441 (match_operand 11 "const_16_to_31_operand")
13409 (match_operand 12 "const_16_to_31_operand") 14442 (match_operand 12 "const_16_to_31_operand")
13410 (match_operand 13 "const_16_to_31_operand") 14443 (match_operand 13 "const_16_to_31_operand")
13411 (match_operand 14 "const_16_to_31_operand") 14444 (match_operand 14 "const_16_to_31_operand")
13412 (match_operand 15 "const_16_to_31_operand") 14445 (match_operand 15 "const_16_to_31_operand")
13413 (match_operand 16 "const_16_to_31_operand") 14446 (match_operand 16 "const_16_to_31_operand")
13414 (match_operand 17 "const_16_to_31_operand") 14447 (match_operand 17 "const_16_to_31_operand")
13415 (match_operand 18 "const_16_to_31_operand")])))] 14448 (match_operand 18 "const_16_to_31_operand")])))]
13416 "TARGET_AVX512F 14449 "TARGET_AVX512F
13417 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1) 14450 && (INTVAL (operands[3]) & 3) == 0
13418 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2) 14451 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
13419 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3) 14452 && INTVAL (operands[3]) == INTVAL (operands[5]) - 2
13420 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1) 14453 && INTVAL (operands[3]) == INTVAL (operands[6]) - 3
13421 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2) 14454 && (INTVAL (operands[7]) & 3) == 0
13422 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3) 14455 && INTVAL (operands[7]) == INTVAL (operands[8]) - 1
13423 && INTVAL (operands[11]) == (INTVAL (operands[12]) - 1) 14456 && INTVAL (operands[7]) == INTVAL (operands[9]) - 2
13424 && INTVAL (operands[11]) == (INTVAL (operands[13]) - 2) 14457 && INTVAL (operands[7]) == INTVAL (operands[10]) - 3
13425 && INTVAL (operands[11]) == (INTVAL (operands[14]) - 3) 14458 && (INTVAL (operands[11]) & 3) == 0
13426 && INTVAL (operands[15]) == (INTVAL (operands[16]) - 1) 14459 && INTVAL (operands[11]) == INTVAL (operands[12]) - 1
13427 && INTVAL (operands[15]) == (INTVAL (operands[17]) - 2) 14460 && INTVAL (operands[11]) == INTVAL (operands[13]) - 2
13428 && INTVAL (operands[15]) == (INTVAL (operands[18]) - 3))" 14461 && INTVAL (operands[11]) == INTVAL (operands[14]) - 3
14462 && (INTVAL (operands[15]) & 3) == 0
14463 && INTVAL (operands[15]) == INTVAL (operands[16]) - 1
14464 && INTVAL (operands[15]) == INTVAL (operands[17]) - 2
14465 && INTVAL (operands[15]) == INTVAL (operands[18]) - 3"
13429 { 14466 {
13430 int mask; 14467 int mask;
13431 mask = INTVAL (operands[3]) / 4; 14468 mask = INTVAL (operands[3]) / 4;
13432 mask |= INTVAL (operands[7]) / 4 << 2; 14469 mask |= INTVAL (operands[7]) / 4 << 2;
13433 mask |= (INTVAL (operands[11]) - 16) / 4 << 4; 14470 mask |= (INTVAL (operands[11]) - 16) / 4 << 4;
14050 (const_string "1"))) 15087 (const_string "1")))
14051 (set_attr "length_immediate" "1") 15088 (set_attr "length_immediate" "1")
14052 (set_attr "prefix" "maybe_vex") 15089 (set_attr "prefix" "maybe_vex")
14053 (set_attr "mode" "TI")]) 15090 (set_attr "mode" "TI")])
14054 15091
15092 (define_insn "*vec_extractv16qi_zext"
15093 [(set (match_operand:HI 0 "register_operand" "=r,r")
15094 (zero_extend:HI
15095 (vec_select:QI
15096 (match_operand:V16QI 1 "register_operand" "x,v")
15097 (parallel
15098 [(match_operand:SI 2 "const_0_to_15_operand")]))))]
15099 "TARGET_SSE4_1"
15100 "@
15101 %vpextrb\t{%2, %1, %k0|%k0, %1, %2}
15102 vpextrb\t{%2, %1, %k0|%k0, %1, %2}"
15103 [(set_attr "isa" "*,avx512bw")
15104 (set_attr "type" "sselog1")
15105 (set_attr "prefix_data16" "1")
15106 (set_attr "prefix_extra" "1")
15107 (set_attr "length_immediate" "1")
15108 (set_attr "prefix" "maybe_vex")
15109 (set_attr "mode" "TI")])
15110
14055 (define_insn "*vec_extract<mode>_mem" 15111 (define_insn "*vec_extract<mode>_mem"
14056 [(set (match_operand:<ssescalarmode> 0 "register_operand" "=r") 15112 [(set (match_operand:<ssescalarmode> 0 "register_operand" "=r")
14057 (vec_select:<ssescalarmode> 15113 (vec_select:<ssescalarmode>
14058 (match_operand:VI12_128 1 "memory_operand" "o") 15114 (match_operand:VI12_128 1 "memory_operand" "o")
14059 (parallel 15115 (parallel
14074 (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC") 15130 (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
14075 ] 15131 ]
14076 (symbol_ref "true")))]) 15132 (symbol_ref "true")))])
14077 15133
14078 (define_insn "*vec_extractv2di_0_sse" 15134 (define_insn "*vec_extractv2di_0_sse"
14079 [(set (match_operand:DI 0 "nonimmediate_operand" "=v,m") 15135 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,x ,m")
14080 (vec_select:DI 15136 (vec_select:DI
14081 (match_operand:V2DI 1 "nonimmediate_operand" "vm,v") 15137 (match_operand:V2DI 1 "nonimmediate_operand" " x,xm,x")
14082 (parallel [(const_int 0)])))] 15138 (parallel [(const_int 0)])))]
14083 "TARGET_SSE && !TARGET_64BIT 15139 "TARGET_SSE && !TARGET_64BIT
14084 && !(MEM_P (operands[0]) && MEM_P (operands[1]))" 15140 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
14085 "#") 15141 "#"
15142 [(set_attr "isa" "sse4,*,*")
15143 (set (attr "preferred_for_speed")
15144 (cond [(eq_attr "alternative" "0")
15145 (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
15146 ]
15147 (symbol_ref "true")))])
15148
15149 (define_split
15150 [(set (match_operand:DI 0 "general_reg_operand")
15151 (vec_select:DI
15152 (match_operand:V2DI 1 "register_operand")
15153 (parallel [(const_int 0)])))]
15154 "TARGET_SSE4_1 && !TARGET_64BIT
15155 && reload_completed"
15156 [(set (match_dup 2) (match_dup 4))
15157 (set (match_dup 3)
15158 (vec_select:SI
15159 (match_dup 5)
15160 (parallel [(const_int 1)])))]
15161 {
15162 operands[4] = gen_lowpart (SImode, operands[1]);
15163 operands[5] = gen_lowpart (V4SImode, operands[1]);
15164 split_double_mode (DImode, &operands[0], 1, &operands[2], &operands[3]);
15165 })
14086 15166
14087 (define_split 15167 (define_split
14088 [(set (match_operand:SWI48x 0 "nonimmediate_operand") 15168 [(set (match_operand:SWI48x 0 "nonimmediate_operand")
14089 (vec_select:SWI48x 15169 (vec_select:SWI48x
14090 (match_operand:<ssevecmode> 1 "register_operand") 15170 (match_operand:<ssevecmode> 1 "register_operand")
14387 vpunpckldq\t{%2, %1, %0|%0, %1, %2} 15467 vpunpckldq\t{%2, %1, %0|%0, %1, %2}
14388 %vmovd\t{%1, %0|%0, %1} 15468 %vmovd\t{%1, %0|%0, %1}
14389 punpckldq\t{%2, %0|%0, %2} 15469 punpckldq\t{%2, %0|%0, %2}
14390 movd\t{%1, %0|%0, %1}" 15470 movd\t{%1, %0|%0, %1}"
14391 [(set_attr "isa" "noavx,noavx,avx,avx512dq,noavx,noavx,avx,*,*,*") 15471 [(set_attr "isa" "noavx,noavx,avx,avx512dq,noavx,noavx,avx,*,*,*")
15472 (set (attr "mmx_isa")
15473 (if_then_else (eq_attr "alternative" "8,9")
15474 (const_string "native")
15475 (const_string "*")))
14392 (set (attr "type") 15476 (set (attr "type")
14393 (cond [(eq_attr "alternative" "7") 15477 (cond [(eq_attr "alternative" "7")
14394 (const_string "ssemov") 15478 (const_string "ssemov")
14395 (eq_attr "alternative" "8") 15479 (eq_attr "alternative" "8")
14396 (const_string "mmxcvt") 15480 (const_string "mmxcvt")
14411 15495
14412 ;; ??? In theory we can match memory for the MMX alternative, but allowing 15496 ;; ??? In theory we can match memory for the MMX alternative, but allowing
14413 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE 15497 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
14414 ;; alternatives pretty much forces the MMX alternative to be chosen. 15498 ;; alternatives pretty much forces the MMX alternative to be chosen.
14415 (define_insn "*vec_concatv2si" 15499 (define_insn "*vec_concatv2si"
14416 [(set (match_operand:V2SI 0 "register_operand" "=x,x ,*y,x,x,*y,*y") 15500 [(set (match_operand:V2SI 0 "register_operand" "=x,x ,x,x,*y,*y")
14417 (vec_concat:V2SI 15501 (vec_concat:V2SI
14418 (match_operand:SI 1 "nonimmediate_operand" " 0,rm,rm,0,m, 0,*rm") 15502 (match_operand:SI 1 "nonimmediate_operand" " 0,rm,0,m, 0,rm")
14419 (match_operand:SI 2 "reg_or_0_operand" " x,C ,C, x,C,*y,C")))] 15503 (match_operand:SI 2 "reg_or_0_operand" " x,C ,x,C,*y,C")))]
14420 "TARGET_SSE && !TARGET_SSE4_1" 15504 "TARGET_SSE && !TARGET_SSE4_1"
14421 "@ 15505 "@
14422 punpckldq\t{%2, %0|%0, %2} 15506 punpckldq\t{%2, %0|%0, %2}
14423 movd\t{%1, %0|%0, %1}
14424 movd\t{%1, %0|%0, %1} 15507 movd\t{%1, %0|%0, %1}
14425 unpcklps\t{%2, %0|%0, %2} 15508 unpcklps\t{%2, %0|%0, %2}
14426 movss\t{%1, %0|%0, %1} 15509 movss\t{%1, %0|%0, %1}
14427 punpckldq\t{%2, %0|%0, %2} 15510 punpckldq\t{%2, %0|%0, %2}
14428 movd\t{%1, %0|%0, %1}" 15511 movd\t{%1, %0|%0, %1}"
14429 [(set_attr "isa" "sse2,sse2,sse2,*,*,*,*") 15512 [(set_attr "isa" "sse2,sse2,*,*,*,*")
14430 (set_attr "type" "sselog,ssemov,mmxmov,sselog,ssemov,mmxcvt,mmxmov") 15513 (set_attr "mmx_isa" "*,*,*,*,native,native")
14431 (set_attr "mode" "TI,TI,DI,V4SF,SF,DI,DI")]) 15514 (set_attr "type" "sselog,ssemov,sselog,ssemov,mmxcvt,mmxmov")
15515 (set_attr "mode" "TI,TI,V4SF,SF,DI,DI")])
14432 15516
14433 (define_insn "*vec_concatv4si" 15517 (define_insn "*vec_concatv4si"
14434 [(set (match_operand:V4SI 0 "register_operand" "=x,v,x,x,v") 15518 [(set (match_operand:V4SI 0 "register_operand" "=x,v,x,x,v")
14435 (vec_concat:V4SI 15519 (vec_concat:V4SI
14436 (match_operand:V2SI 1 "register_operand" " 0,v,0,0,v") 15520 (match_operand:V2SI 1 "register_operand" " 0,v,0,0,v")
14445 [(set_attr "isa" "sse2_noavx,avx,noavx,noavx,avx") 15529 [(set_attr "isa" "sse2_noavx,avx,noavx,noavx,avx")
14446 (set_attr "type" "sselog,sselog,ssemov,ssemov,ssemov") 15530 (set_attr "type" "sselog,sselog,ssemov,ssemov,ssemov")
14447 (set_attr "prefix" "orig,maybe_evex,orig,orig,maybe_evex") 15531 (set_attr "prefix" "orig,maybe_evex,orig,orig,maybe_evex")
14448 (set_attr "mode" "TI,TI,V4SF,V2SF,V2SF")]) 15532 (set_attr "mode" "TI,TI,V4SF,V2SF,V2SF")])
14449 15533
14450 ;; movd instead of movq is required to handle broken assemblers. 15534 (define_insn "*vec_concatv4si_0"
15535 [(set (match_operand:V4SI 0 "register_operand" "=v,x")
15536 (vec_concat:V4SI
15537 (match_operand:V2SI 1 "nonimmediate_operand" "vm,?!*y")
15538 (match_operand:V2SI 2 "const0_operand" " C,C")))]
15539 "TARGET_SSE2"
15540 "@
15541 %vmovq\t{%1, %0|%0, %1}
15542 movq2dq\t{%1, %0|%0, %1}"
15543 [(set_attr "mmx_isa" "*,native")
15544 (set_attr "type" "ssemov")
15545 (set_attr "prefix" "maybe_vex,orig")
15546 (set_attr "mode" "TI")])
15547
14451 (define_insn "vec_concatv2di" 15548 (define_insn "vec_concatv2di"
14452 [(set (match_operand:V2DI 0 "register_operand" 15549 [(set (match_operand:V2DI 0 "register_operand"
14453 "=Yr,*x,x ,v ,v,v ,x ,x,v ,x,x,v") 15550 "=Yr,*x,x ,v ,x,v ,x,x,v")
14454 (vec_concat:V2DI 15551 (vec_concat:V2DI
14455 (match_operand:DI 1 "nonimmediate_operand" 15552 (match_operand:DI 1 "register_operand"
14456 " 0, 0,x ,Yv,r,vm,?!*y,0,Yv,0,0,v") 15553 " 0, 0,x ,Yv,0,Yv,0,0,v")
14457 (match_operand:DI 2 "nonimm_or_0_operand" 15554 (match_operand:DI 2 "nonimmediate_operand"
14458 " rm,rm,rm,rm,C ,C ,C ,x,Yv,x,m,m")))] 15555 " rm,rm,rm,rm,x,Yv,x,m,m")))]
14459 "TARGET_SSE" 15556 "TARGET_SSE"
14460 "@ 15557 "@
14461 pinsrq\t{$1, %2, %0|%0, %2, 1} 15558 pinsrq\t{$1, %2, %0|%0, %2, 1}
14462 pinsrq\t{$1, %2, %0|%0, %2, 1} 15559 pinsrq\t{$1, %2, %0|%0, %2, 1}
14463 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1} 15560 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
14464 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1} 15561 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
14465 * return HAVE_AS_IX86_INTERUNIT_MOVQ ? \"%vmovq\t{%1, %0|%0, %1}\" : \"%vmovd\t{%1, %0|%0, %1}\";
14466 %vmovq\t{%1, %0|%0, %1}
14467 movq2dq\t{%1, %0|%0, %1}
14468 punpcklqdq\t{%2, %0|%0, %2} 15562 punpcklqdq\t{%2, %0|%0, %2}
14469 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2} 15563 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
14470 movlhps\t{%2, %0|%0, %2} 15564 movlhps\t{%2, %0|%0, %2}
14471 movhps\t{%2, %0|%0, %2} 15565 movhps\t{%2, %0|%0, %2}
14472 vmovhps\t{%2, %1, %0|%0, %1, %2}" 15566 vmovhps\t{%2, %1, %0|%0, %1, %2}"
14476 (eq_attr "alternative" "2") 15570 (eq_attr "alternative" "2")
14477 (const_string "x64_avx") 15571 (const_string "x64_avx")
14478 (eq_attr "alternative" "3") 15572 (eq_attr "alternative" "3")
14479 (const_string "x64_avx512dq") 15573 (const_string "x64_avx512dq")
14480 (eq_attr "alternative" "4") 15574 (eq_attr "alternative" "4")
14481 (const_string "x64_sse2")
14482 (eq_attr "alternative" "5,6")
14483 (const_string "sse2")
14484 (eq_attr "alternative" "7")
14485 (const_string "sse2_noavx") 15575 (const_string "sse2_noavx")
14486 (eq_attr "alternative" "8,11") 15576 (eq_attr "alternative" "5,8")
14487 (const_string "avx") 15577 (const_string "avx")
14488 ] 15578 ]
14489 (const_string "noavx"))) 15579 (const_string "noavx")))
14490 (set (attr "type") 15580 (set (attr "type")
14491 (if_then_else 15581 (if_then_else
14492 (eq_attr "alternative" "0,1,2,3,7,8") 15582 (eq_attr "alternative" "0,1,2,3,4,5")
14493 (const_string "sselog") 15583 (const_string "sselog")
14494 (const_string "ssemov"))) 15584 (const_string "ssemov")))
14495 (set (attr "prefix_rex") 15585 (set (attr "prefix_rex")
14496 (if_then_else (eq_attr "alternative" "0,1,2,3,4") 15586 (if_then_else (eq_attr "alternative" "0,1,2,3")
14497 (const_string "1") 15587 (const_string "1")
14498 (const_string "*"))) 15588 (const_string "*")))
14499 (set (attr "prefix_extra") 15589 (set (attr "prefix_extra")
14500 (if_then_else (eq_attr "alternative" "0,1,2,3") 15590 (if_then_else (eq_attr "alternative" "0,1,2,3")
14501 (const_string "1") 15591 (const_string "1")
14507 (set (attr "prefix") 15597 (set (attr "prefix")
14508 (cond [(eq_attr "alternative" "2") 15598 (cond [(eq_attr "alternative" "2")
14509 (const_string "vex") 15599 (const_string "vex")
14510 (eq_attr "alternative" "3") 15600 (eq_attr "alternative" "3")
14511 (const_string "evex") 15601 (const_string "evex")
14512 (eq_attr "alternative" "4,5") 15602 (eq_attr "alternative" "5,8")
14513 (const_string "maybe_vex")
14514 (eq_attr "alternative" "8,11")
14515 (const_string "maybe_evex") 15603 (const_string "maybe_evex")
14516 ] 15604 ]
14517 (const_string "orig"))) 15605 (const_string "orig")))
14518 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF") 15606 (set_attr "mode" "TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
15607
15608 (define_insn "*vec_concatv2di_0"
15609 [(set (match_operand:V2DI 0 "register_operand" "=v,v ,x")
15610 (vec_concat:V2DI
15611 (match_operand:DI 1 "nonimmediate_operand" " r,vm,?!*y")
15612 (match_operand:DI 2 "const0_operand" " C,C ,C")))]
15613 "TARGET_SSE2"
15614 "@
15615 * return HAVE_AS_IX86_INTERUNIT_MOVQ ? \"%vmovq\t{%1, %0|%0, %1}\" : \"%vmovd\t{%1, %0|%0, %1}\";
15616 %vmovq\t{%1, %0|%0, %1}
15617 movq2dq\t{%1, %0|%0, %1}"
15618 [(set_attr "isa" "x64,*,*")
15619 (set_attr "mmx_isa" "*,*,native")
15620 (set_attr "type" "ssemov")
15621 (set_attr "prefix_rex" "1,*,*")
15622 (set_attr "prefix" "maybe_vex,maybe_vex,orig")
15623 (set_attr "mode" "TI")
14519 (set (attr "preferred_for_speed") 15624 (set (attr "preferred_for_speed")
14520 (cond [(eq_attr "alternative" "4") 15625 (cond [(eq_attr "alternative" "0")
14521 (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC") 15626 (symbol_ref "TARGET_INTER_UNIT_MOVES_TO_VEC")
14522 (eq_attr "alternative" "6")
14523 (symbol_ref "TARGET_INTER_UNIT_MOVES_FROM_VEC")
14524 ] 15627 ]
14525 (symbol_ref "true")))]) 15628 (symbol_ref "true")))])
14526 15629
14527 ;; vmovq clears also the higher bits. 15630 ;; vmovq clears also the higher bits.
14528 (define_insn "vec_set<mode>_0" 15631 (define_insn "vec_set<mode>_0"
14529 [(set (match_operand:VI8_AVX_AVX512F 0 "register_operand" "=v,v") 15632 [(set (match_operand:VI8_AVX_AVX512F 0 "register_operand" "=v,v")
14530 (vec_merge:VI8_AVX_AVX512F 15633 (vec_merge:VI8_AVX_AVX512F
14531 (vec_duplicate:VI8_AVX_AVX512F 15634 (vec_duplicate:VI8_AVX_AVX512F
14532 (match_operand:<ssescalarmode> 2 "general_operand" "r,vm")) 15635 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "r,vm"))
14533 (match_operand:VI8_AVX_AVX512F 1 "const0_operand" "C,C") 15636 (match_operand:VI8_AVX_AVX512F 1 "const0_operand" "C,C")
14534 (const_int 1)))] 15637 (const_int 1)))]
14535 "TARGET_AVX" 15638 "TARGET_AVX"
14536 "vmovq\t{%2, %x0|%x0, %2}" 15639 "vmovq\t{%2, %x0|%x0, %2}"
14537 [(set_attr "isa" "x64,*") 15640 [(set_attr "isa" "x64,*")
14561 [(match_operand:<sseunpackmode> 0 "register_operand") 15664 [(match_operand:<sseunpackmode> 0 "register_operand")
14562 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")] 15665 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14563 "TARGET_SSE2" 15666 "TARGET_SSE2"
14564 "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;") 15667 "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
14565 15668
15669 (define_expand "vec_unpacks_sbool_lo_qi"
15670 [(match_operand:QI 0 "register_operand")
15671 (match_operand:QI 1 "register_operand")
15672 (match_operand:QI 2 "const_int_operand")]
15673 "TARGET_AVX512F"
15674 {
15675 if (INTVAL (operands[2]) != 8 && INTVAL (operands[2]) != 4)
15676 FAIL;
15677 emit_move_insn (operands[0], operands[1]);
15678 DONE;
15679 })
15680
14566 (define_expand "vec_unpacks_lo_hi" 15681 (define_expand "vec_unpacks_lo_hi"
14567 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0) 15682 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0)
14568 (match_operand:HI 1 "register_operand"))] 15683 (match_operand:HI 1 "register_operand"))]
14569 "TARGET_AVX512F") 15684 "TARGET_AVX512F")
14570 15685
14581 (define_expand "vec_unpacku_hi_<mode>" 15696 (define_expand "vec_unpacku_hi_<mode>"
14582 [(match_operand:<sseunpackmode> 0 "register_operand") 15697 [(match_operand:<sseunpackmode> 0 "register_operand")
14583 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")] 15698 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14584 "TARGET_SSE2" 15699 "TARGET_SSE2"
14585 "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;") 15700 "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
15701
15702 (define_expand "vec_unpacks_sbool_hi_qi"
15703 [(match_operand:QI 0 "register_operand")
15704 (match_operand:QI 1 "register_operand")
15705 (match_operand:QI 2 "const_int_operand")]
15706 "TARGET_AVX512F"
15707 {
15708 HOST_WIDE_INT nunits = INTVAL (operands[2]);
15709 if (nunits != 8 && nunits != 4)
15710 FAIL;
15711 if (TARGET_AVX512DQ)
15712 emit_insn (gen_klshiftrtqi (operands[0], operands[1],
15713 GEN_INT (nunits / 2)));
15714 else
15715 {
15716 rtx tem = gen_reg_rtx (HImode);
15717 emit_insn (gen_klshiftrthi (tem, lowpart_subreg (HImode, operands[1],
15718 QImode),
15719 GEN_INT (nunits / 2)));
15720 emit_move_insn (operands[0], lowpart_subreg (QImode, tem, HImode));
15721 }
15722 DONE;
15723 })
14586 15724
14587 (define_expand "vec_unpacks_hi_hi" 15725 (define_expand "vec_unpacks_hi_hi"
14588 [(parallel 15726 [(parallel
14589 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0) 15727 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0)
14590 (lshiftrt:HI (match_operand:HI 1 "register_operand") 15728 (lshiftrt:HI (match_operand:HI 1 "register_operand")
14620 (match_operand:VI12_AVX2 2 "vector_operand"))) 15758 (match_operand:VI12_AVX2 2 "vector_operand")))
14621 (match_dup <mask_expand_op3>)) 15759 (match_dup <mask_expand_op3>))
14622 (const_int 1))))] 15760 (const_int 1))))]
14623 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>" 15761 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
14624 { 15762 {
14625 operands[<mask_expand_op3>] = CONST1_RTX(<MODE>mode); 15763 operands[<mask_expand_op3>] = CONST1_RTX(<ssedoublemode>mode);
14626 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands); 15764 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
14627 }) 15765 })
14628 15766
14629 (define_insn "*<sse2_avx2>_uavg<mode>3<mask_name>" 15767 (define_insn "*<sse2_avx2>_uavg<mode>3<mask_name>"
14630 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v") 15768 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v")
14634 (plus:<ssedoublemode> 15772 (plus:<ssedoublemode>
14635 (zero_extend:<ssedoublemode> 15773 (zero_extend:<ssedoublemode>
14636 (match_operand:VI12_AVX2 1 "vector_operand" "%0,v")) 15774 (match_operand:VI12_AVX2 1 "vector_operand" "%0,v"))
14637 (zero_extend:<ssedoublemode> 15775 (zero_extend:<ssedoublemode>
14638 (match_operand:VI12_AVX2 2 "vector_operand" "xBm,vm"))) 15776 (match_operand:VI12_AVX2 2 "vector_operand" "xBm,vm")))
14639 (match_operand:VI12_AVX2 <mask_expand_op3> "const1_operand")) 15777 (match_operand:<ssedoublemode> <mask_expand_op3> "const1_operand"))
14640 (const_int 1))))] 15778 (const_int 1))))]
14641 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition> 15779 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
14642 && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 15780 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
14643 "@ 15781 "@
14644 pavg<ssemodesuffix>\t{%2, %0|%0, %2} 15782 pavg<ssemodesuffix>\t{%2, %0|%0, %2}
14677 "%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}" 15815 "%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}"
14678 [(set_attr "type" "ssemov") 15816 [(set_attr "type" "ssemov")
14679 (set_attr "prefix" "maybe_vex") 15817 (set_attr "prefix" "maybe_vex")
14680 (set_attr "mode" "<MODE>")]) 15818 (set_attr "mode" "<MODE>")])
14681 15819
14682 (define_insn "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_zext" 15820 (define_insn "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_<u>ext"
14683 [(set (match_operand:DI 0 "register_operand" "=r") 15821 [(set (match_operand:DI 0 "register_operand" "=r")
14684 (zero_extend:DI 15822 (any_extend:DI
14685 (unspec:SI 15823 (unspec:SI
14686 [(match_operand:VF_128_256 1 "register_operand" "x")] 15824 [(match_operand:VF_128_256 1 "register_operand" "x")]
14687 UNSPEC_MOVMSK)))] 15825 UNSPEC_MOVMSK)))]
14688 "TARGET_64BIT && TARGET_SSE" 15826 "TARGET_64BIT && TARGET_SSE"
14689 "%vmovmsk<ssemodesuffix>\t{%1, %k0|%k0, %1}" 15827 "%vmovmsk<ssemodesuffix>\t{%1, %k0|%k0, %1}"
15828 [(set_attr "type" "ssemov")
15829 (set_attr "prefix" "maybe_vex")
15830 (set_attr "mode" "<MODE>")])
15831
15832 (define_insn_and_split "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_lt"
15833 [(set (match_operand:SI 0 "register_operand" "=r")
15834 (unspec:SI
15835 [(lt:VF_128_256
15836 (match_operand:<sseintvecmode> 1 "register_operand" "x")
15837 (match_operand:<sseintvecmode> 2 "const0_operand" "C"))]
15838 UNSPEC_MOVMSK))]
15839 "TARGET_SSE"
15840 "#"
15841 "&& reload_completed"
15842 [(set (match_dup 0)
15843 (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK))]
15844 "operands[1] = gen_lowpart (<MODE>mode, operands[1]);"
15845 [(set_attr "type" "ssemov")
15846 (set_attr "prefix" "maybe_vex")
15847 (set_attr "mode" "<MODE>")])
15848
15849 (define_insn_and_split "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_<u>ext_lt"
15850 [(set (match_operand:DI 0 "register_operand" "=r")
15851 (any_extend:DI
15852 (unspec:SI
15853 [(lt:VF_128_256
15854 (match_operand:<sseintvecmode> 1 "register_operand" "x")
15855 (match_operand:<sseintvecmode> 2 "const0_operand" "C"))]
15856 UNSPEC_MOVMSK)))]
15857 "TARGET_64BIT && TARGET_SSE"
15858 "#"
15859 "&& reload_completed"
15860 [(set (match_dup 0)
15861 (any_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK)))]
15862 "operands[1] = gen_lowpart (<MODE>mode, operands[1]);"
15863 [(set_attr "type" "ssemov")
15864 (set_attr "prefix" "maybe_vex")
15865 (set_attr "mode" "<MODE>")])
15866
15867 (define_insn_and_split "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_shift"
15868 [(set (match_operand:SI 0 "register_operand" "=r")
15869 (unspec:SI
15870 [(subreg:VF_128_256
15871 (ashiftrt:<sseintvecmode>
15872 (match_operand:<sseintvecmode> 1 "register_operand" "x")
15873 (match_operand:QI 2 "const_int_operand" "n")) 0)]
15874 UNSPEC_MOVMSK))]
15875 "TARGET_SSE"
15876 "#"
15877 "&& reload_completed"
15878 [(set (match_dup 0)
15879 (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK))]
15880 "operands[1] = gen_lowpart (<MODE>mode, operands[1]);"
15881 [(set_attr "type" "ssemov")
15882 (set_attr "prefix" "maybe_vex")
15883 (set_attr "mode" "<MODE>")])
15884
15885 (define_insn_and_split "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_<u>ext_shift"
15886 [(set (match_operand:DI 0 "register_operand" "=r")
15887 (any_extend:DI
15888 (unspec:SI
15889 [(subreg:VF_128_256
15890 (ashiftrt:<sseintvecmode>
15891 (match_operand:<sseintvecmode> 1 "register_operand" "x")
15892 (match_operand:QI 2 "const_int_operand" "n")) 0)]
15893 UNSPEC_MOVMSK)))]
15894 "TARGET_64BIT && TARGET_SSE"
15895 "#"
15896 "&& reload_completed"
15897 [(set (match_dup 0)
15898 (any_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK)))]
15899 "operands[1] = gen_lowpart (<MODE>mode, operands[1]);"
14690 [(set_attr "type" "ssemov") 15900 [(set_attr "type" "ssemov")
14691 (set_attr "prefix" "maybe_vex") 15901 (set_attr "prefix" "maybe_vex")
14692 (set_attr "mode" "<MODE>")]) 15902 (set_attr "mode" "<MODE>")])
14693 15903
14694 (define_insn "<sse2_avx2>_pmovmskb" 15904 (define_insn "<sse2_avx2>_pmovmskb"
14722 (const_string "*") 15932 (const_string "*")
14723 (const_string "1"))) 15933 (const_string "1")))
14724 (set_attr "prefix" "maybe_vex") 15934 (set_attr "prefix" "maybe_vex")
14725 (set_attr "mode" "SI")]) 15935 (set_attr "mode" "SI")])
14726 15936
15937 (define_insn "*sse2_pmovmskb_ext"
15938 [(set (match_operand:DI 0 "register_operand" "=r")
15939 (sign_extend:DI
15940 (unspec:SI
15941 [(match_operand:V16QI 1 "register_operand" "x")]
15942 UNSPEC_MOVMSK)))]
15943 "TARGET_64BIT && TARGET_SSE2"
15944 "%vpmovmskb\t{%1, %k0|%k0, %1}"
15945 [(set_attr "type" "ssemov")
15946 (set (attr "prefix_data16")
15947 (if_then_else
15948 (match_test "TARGET_AVX")
15949 (const_string "*")
15950 (const_string "1")))
15951 (set_attr "prefix" "maybe_vex")
15952 (set_attr "mode" "SI")])
15953
15954 (define_insn_and_split "*<sse2_avx2>_pmovmskb_lt"
15955 [(set (match_operand:SI 0 "register_operand" "=r")
15956 (unspec:SI
15957 [(lt:VI1_AVX2 (match_operand:VI1_AVX2 1 "register_operand" "x")
15958 (match_operand:VI1_AVX2 2 "const0_operand" "C"))]
15959 UNSPEC_MOVMSK))]
15960 "TARGET_SSE2"
15961 "#"
15962 ""
15963 [(set (match_dup 0)
15964 (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK))]
15965 ""
15966 [(set_attr "type" "ssemov")
15967 (set (attr "prefix_data16")
15968 (if_then_else
15969 (match_test "TARGET_AVX")
15970 (const_string "*")
15971 (const_string "1")))
15972 (set_attr "prefix" "maybe_vex")
15973 (set_attr "mode" "SI")])
15974
15975 (define_insn_and_split "*<sse2_avx2>_pmovmskb_zext_lt"
15976 [(set (match_operand:DI 0 "register_operand" "=r")
15977 (zero_extend:DI
15978 (unspec:SI
15979 [(lt:VI1_AVX2 (match_operand:VI1_AVX2 1 "register_operand" "x")
15980 (match_operand:VI1_AVX2 2 "const0_operand" "C"))]
15981 UNSPEC_MOVMSK)))]
15982 "TARGET_64BIT && TARGET_SSE2"
15983 "#"
15984 ""
15985 [(set (match_dup 0)
15986 (zero_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK)))]
15987 ""
15988 [(set_attr "type" "ssemov")
15989 (set (attr "prefix_data16")
15990 (if_then_else
15991 (match_test "TARGET_AVX")
15992 (const_string "*")
15993 (const_string "1")))
15994 (set_attr "prefix" "maybe_vex")
15995 (set_attr "mode" "SI")])
15996
15997 (define_insn_and_split "*sse2_pmovmskb_ext_lt"
15998 [(set (match_operand:DI 0 "register_operand" "=r")
15999 (sign_extend:DI
16000 (unspec:SI
16001 [(lt:V16QI (match_operand:V16QI 1 "register_operand" "x")
16002 (match_operand:V16QI 2 "const0_operand" "C"))]
16003 UNSPEC_MOVMSK)))]
16004 "TARGET_64BIT && TARGET_SSE2"
16005 "#"
16006 ""
16007 [(set (match_dup 0)
16008 (sign_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_MOVMSK)))]
16009 ""
16010 [(set_attr "type" "ssemov")
16011 (set (attr "prefix_data16")
16012 (if_then_else
16013 (match_test "TARGET_AVX")
16014 (const_string "*")
16015 (const_string "1")))
16016 (set_attr "prefix" "maybe_vex")
16017 (set_attr "mode" "SI")])
16018
14727 (define_expand "sse2_maskmovdqu" 16019 (define_expand "sse2_maskmovdqu"
14728 [(set (match_operand:V16QI 0 "memory_operand") 16020 [(set (match_operand:V16QI 0 "memory_operand")
14729 (unspec:V16QI [(match_operand:V16QI 1 "register_operand") 16021 (unspec:V16QI [(match_operand:V16QI 1 "register_operand")
14730 (match_operand:V16QI 2 "register_operand") 16022 (match_operand:V16QI 2 "register_operand")
14731 (match_dup 0)] 16023 (match_dup 0)]
14798 ;; Since 32bit register operands are implicitly zero extended to 64bit, 16090 ;; Since 32bit register operands are implicitly zero extended to 64bit,
14799 ;; we only need to set up 32bit registers. 16091 ;; we only need to set up 32bit registers.
14800 "mwait" 16092 "mwait"
14801 [(set_attr "length" "3")]) 16093 [(set_attr "length" "3")])
14802 16094
14803 (define_insn "sse3_monitor_<mode>" 16095 (define_insn "@sse3_monitor_<mode>"
14804 [(unspec_volatile [(match_operand:P 0 "register_operand" "a") 16096 [(unspec_volatile [(match_operand:P 0 "register_operand" "a")
14805 (match_operand:SI 1 "register_operand" "c") 16097 (match_operand:SI 1 "register_operand" "c")
14806 (match_operand:SI 2 "register_operand" "d")] 16098 (match_operand:SI 2 "register_operand" "d")]
14807 UNSPECV_MONITOR)] 16099 UNSPECV_MONITOR)]
14808 "TARGET_SSE3" 16100 "TARGET_SSE3"
14944 (set_attr "prefix_data16" "1,*") 16236 (set_attr "prefix_data16" "1,*")
14945 (set_attr "prefix_extra" "1") 16237 (set_attr "prefix_extra" "1")
14946 (set_attr "prefix" "orig,vex") 16238 (set_attr "prefix" "orig,vex")
14947 (set_attr "mode" "TI")]) 16239 (set_attr "mode" "TI")])
14948 16240
14949 (define_insn "ssse3_ph<plusminus_mnemonic>wv4hi3" 16241 (define_insn_and_split "ssse3_ph<plusminus_mnemonic>wv4hi3"
14950 [(set (match_operand:V4HI 0 "register_operand" "=y") 16242 [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yv")
14951 (vec_concat:V4HI 16243 (vec_concat:V4HI
14952 (vec_concat:V2HI 16244 (vec_concat:V2HI
14953 (ssse3_plusminus:HI 16245 (ssse3_plusminus:HI
14954 (vec_select:HI 16246 (vec_select:HI
14955 (match_operand:V4HI 1 "register_operand" "0") 16247 (match_operand:V4HI 1 "register_operand" "0,0,Yv")
14956 (parallel [(const_int 0)])) 16248 (parallel [(const_int 0)]))
14957 (vec_select:HI (match_dup 1) (parallel [(const_int 1)]))) 16249 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
14958 (ssse3_plusminus:HI 16250 (ssse3_plusminus:HI
14959 (vec_select:HI (match_dup 1) (parallel [(const_int 2)])) 16251 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
14960 (vec_select:HI (match_dup 1) (parallel [(const_int 3)])))) 16252 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
14961 (vec_concat:V2HI 16253 (vec_concat:V2HI
14962 (ssse3_plusminus:HI 16254 (ssse3_plusminus:HI
14963 (vec_select:HI 16255 (vec_select:HI
14964 (match_operand:V4HI 2 "nonimmediate_operand" "ym") 16256 (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yv")
14965 (parallel [(const_int 0)])) 16257 (parallel [(const_int 0)]))
14966 (vec_select:HI (match_dup 2) (parallel [(const_int 1)]))) 16258 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
14967 (ssse3_plusminus:HI 16259 (ssse3_plusminus:HI
14968 (vec_select:HI (match_dup 2) (parallel [(const_int 2)])) 16260 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
14969 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))] 16261 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
14970 "TARGET_SSSE3" 16262 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
14971 "ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}" 16263 "@
14972 [(set_attr "type" "sseiadd") 16264 ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}
16265 #
16266 #"
16267 "TARGET_SSSE3 && reload_completed
16268 && SSE_REGNO_P (REGNO (operands[0]))"
16269 [(const_int 0)]
16270 {
16271 /* Generate SSE version of the operation. */
16272 rtx op0 = lowpart_subreg (V8HImode, operands[0],
16273 GET_MODE (operands[0]));
16274 rtx op1 = lowpart_subreg (V8HImode, operands[1],
16275 GET_MODE (operands[1]));
16276 rtx op2 = lowpart_subreg (V8HImode, operands[2],
16277 GET_MODE (operands[2]));
16278 emit_insn (gen_ssse3_ph<plusminus_mnemonic>wv8hi3 (op0, op1, op2));
16279 ix86_move_vector_high_sse_to_mmx (op0);
16280 DONE;
16281 }
16282 [(set_attr "mmx_isa" "native,sse_noavx,avx")
16283 (set_attr "type" "sseiadd")
14973 (set_attr "atom_unit" "complex") 16284 (set_attr "atom_unit" "complex")
14974 (set_attr "prefix_extra" "1") 16285 (set_attr "prefix_extra" "1")
14975 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16286 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14976 (set_attr "mode" "DI")]) 16287 (set_attr "mode" "DI,TI,TI")])
14977 16288
14978 (define_insn "avx2_ph<plusminus_mnemonic>dv8si3" 16289 (define_insn "avx2_ph<plusminus_mnemonic>dv8si3"
14979 [(set (match_operand:V8SI 0 "register_operand" "=x") 16290 [(set (match_operand:V8SI 0 "register_operand" "=x")
14980 (vec_concat:V8SI 16291 (vec_concat:V8SI
14981 (vec_concat:V4SI 16292 (vec_concat:V4SI
15050 (set_attr "prefix_data16" "1,*") 16361 (set_attr "prefix_data16" "1,*")
15051 (set_attr "prefix_extra" "1") 16362 (set_attr "prefix_extra" "1")
15052 (set_attr "prefix" "orig,vex") 16363 (set_attr "prefix" "orig,vex")
15053 (set_attr "mode" "TI")]) 16364 (set_attr "mode" "TI")])
15054 16365
15055 (define_insn "ssse3_ph<plusminus_mnemonic>dv2si3" 16366 (define_insn_and_split "ssse3_ph<plusminus_mnemonic>dv2si3"
15056 [(set (match_operand:V2SI 0 "register_operand" "=y") 16367 [(set (match_operand:V2SI 0 "register_operand" "=y,x,Yv")
15057 (vec_concat:V2SI 16368 (vec_concat:V2SI
15058 (plusminus:SI 16369 (plusminus:SI
15059 (vec_select:SI 16370 (vec_select:SI
15060 (match_operand:V2SI 1 "register_operand" "0") 16371 (match_operand:V2SI 1 "register_operand" "0,0,Yv")
15061 (parallel [(const_int 0)])) 16372 (parallel [(const_int 0)]))
15062 (vec_select:SI (match_dup 1) (parallel [(const_int 1)]))) 16373 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
15063 (plusminus:SI 16374 (plusminus:SI
15064 (vec_select:SI 16375 (vec_select:SI
15065 (match_operand:V2SI 2 "nonimmediate_operand" "ym") 16376 (match_operand:V2SI 2 "register_mmxmem_operand" "ym,x,Yv")
15066 (parallel [(const_int 0)])) 16377 (parallel [(const_int 0)]))
15067 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))] 16378 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
15068 "TARGET_SSSE3" 16379 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15069 "ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}" 16380 "@
15070 [(set_attr "type" "sseiadd") 16381 ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}
16382 #
16383 #"
16384 "TARGET_SSSE3 && reload_completed
16385 && SSE_REGNO_P (REGNO (operands[0]))"
16386 [(const_int 0)]
16387 {
16388 /* Generate SSE version of the operation. */
16389 rtx op0 = lowpart_subreg (V4SImode, operands[0],
16390 GET_MODE (operands[0]));
16391 rtx op1 = lowpart_subreg (V4SImode, operands[1],
16392 GET_MODE (operands[1]));
16393 rtx op2 = lowpart_subreg (V4SImode, operands[2],
16394 GET_MODE (operands[2]));
16395 emit_insn (gen_ssse3_ph<plusminus_mnemonic>dv4si3 (op0, op1, op2));
16396 ix86_move_vector_high_sse_to_mmx (op0);
16397 DONE;
16398 }
16399 [(set_attr "mmx_isa" "native,sse_noavx,avx")
16400 (set_attr "type" "sseiadd")
15071 (set_attr "atom_unit" "complex") 16401 (set_attr "atom_unit" "complex")
15072 (set_attr "prefix_extra" "1") 16402 (set_attr "prefix_extra" "1")
15073 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16403 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15074 (set_attr "mode" "DI")]) 16404 (set_attr "mode" "DI,TI,TI")])
15075 16405
15076 (define_insn "avx2_pmaddubsw256" 16406 (define_insn "avx2_pmaddubsw256"
15077 [(set (match_operand:V16HI 0 "register_operand" "=x,v") 16407 [(set (match_operand:V16HI 0 "register_operand" "=x,v")
15078 (ss_plus:V16HI 16408 (ss_plus:V16HI
15079 (mult:V16HI 16409 (mult:V16HI
15220 (set_attr "prefix_extra" "1") 16550 (set_attr "prefix_extra" "1")
15221 (set_attr "prefix" "orig,vex,evex") 16551 (set_attr "prefix" "orig,vex,evex")
15222 (set_attr "mode" "TI")]) 16552 (set_attr "mode" "TI")])
15223 16553
15224 (define_insn "ssse3_pmaddubsw" 16554 (define_insn "ssse3_pmaddubsw"
15225 [(set (match_operand:V4HI 0 "register_operand" "=y") 16555 [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yv")
15226 (ss_plus:V4HI 16556 (ss_plus:V4HI
15227 (mult:V4HI 16557 (mult:V4HI
15228 (zero_extend:V4HI 16558 (zero_extend:V4HI
15229 (vec_select:V4QI 16559 (vec_select:V4QI
15230 (match_operand:V8QI 1 "register_operand" "0") 16560 (match_operand:V8QI 1 "register_operand" "0,0,Yv")
15231 (parallel [(const_int 0) (const_int 2) 16561 (parallel [(const_int 0) (const_int 2)
15232 (const_int 4) (const_int 6)]))) 16562 (const_int 4) (const_int 6)])))
15233 (sign_extend:V4HI 16563 (sign_extend:V4HI
15234 (vec_select:V4QI 16564 (vec_select:V4QI
15235 (match_operand:V8QI 2 "nonimmediate_operand" "ym") 16565 (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yv")
15236 (parallel [(const_int 0) (const_int 2) 16566 (parallel [(const_int 0) (const_int 2)
15237 (const_int 4) (const_int 6)])))) 16567 (const_int 4) (const_int 6)]))))
15238 (mult:V4HI 16568 (mult:V4HI
15239 (zero_extend:V4HI 16569 (zero_extend:V4HI
15240 (vec_select:V4QI (match_dup 1) 16570 (vec_select:V4QI (match_dup 1)
15242 (const_int 5) (const_int 7)]))) 16572 (const_int 5) (const_int 7)])))
15243 (sign_extend:V4HI 16573 (sign_extend:V4HI
15244 (vec_select:V4QI (match_dup 2) 16574 (vec_select:V4QI (match_dup 2)
15245 (parallel [(const_int 1) (const_int 3) 16575 (parallel [(const_int 1) (const_int 3)
15246 (const_int 5) (const_int 7)]))))))] 16576 (const_int 5) (const_int 7)]))))))]
15247 "TARGET_SSSE3" 16577 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15248 "pmaddubsw\t{%2, %0|%0, %2}" 16578 "@
15249 [(set_attr "type" "sseiadd") 16579 pmaddubsw\t{%2, %0|%0, %2}
16580 pmaddubsw\t{%2, %0|%0, %2}
16581 vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
16582 [(set_attr "isa" "*,noavx,avx")
16583 (set_attr "mmx_isa" "native,*,*")
16584 (set_attr "type" "sseiadd")
15250 (set_attr "atom_unit" "simul") 16585 (set_attr "atom_unit" "simul")
15251 (set_attr "prefix_extra" "1") 16586 (set_attr "prefix_extra" "1")
15252 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16587 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15253 (set_attr "mode" "DI")]) 16588 (set_attr "mode" "DI,TI,TI")])
15254 16589
15255 (define_mode_iterator PMULHRSW 16590 (define_mode_iterator PMULHRSW
15256 [V4HI V8HI (V16HI "TARGET_AVX2")]) 16591 [V8HI (V16HI "TARGET_AVX2")])
15257 16592
15258 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3_mask" 16593 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3_mask"
15259 [(set (match_operand:PMULHRSW 0 "register_operand") 16594 [(set (match_operand:PMULHRSW 0 "register_operand")
15260 (vec_merge:PMULHRSW 16595 (vec_merge:PMULHRSW
15261 (truncate:PMULHRSW 16596 (truncate:PMULHRSW
15290 (sign_extend:<ssedoublemode> 16625 (sign_extend:<ssedoublemode>
15291 (match_operand:PMULHRSW 2 "nonimmediate_operand"))) 16626 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
15292 (const_int 14)) 16627 (const_int 14))
15293 (match_dup 3)) 16628 (match_dup 3))
15294 (const_int 1))))] 16629 (const_int 1))))]
15295 "TARGET_AVX2" 16630 "TARGET_SSSE3"
16631 {
16632 operands[3] = CONST1_RTX(<MODE>mode);
16633 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
16634 })
16635
16636 (define_expand "smulhrs<mode>3"
16637 [(set (match_operand:VI2_AVX2 0 "register_operand")
16638 (truncate:VI2_AVX2
16639 (lshiftrt:<ssedoublemode>
16640 (plus:<ssedoublemode>
16641 (lshiftrt:<ssedoublemode>
16642 (mult:<ssedoublemode>
16643 (sign_extend:<ssedoublemode>
16644 (match_operand:VI2_AVX2 1 "nonimmediate_operand"))
16645 (sign_extend:<ssedoublemode>
16646 (match_operand:VI2_AVX2 2 "nonimmediate_operand")))
16647 (const_int 14))
16648 (match_dup 3))
16649 (const_int 1))))]
16650 "TARGET_SSSE3"
15296 { 16651 {
15297 operands[3] = CONST1_RTX(<MODE>mode); 16652 operands[3] = CONST1_RTX(<MODE>mode);
15298 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands); 16653 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
15299 }) 16654 })
15300 16655
15323 (set_attr "prefix_data16" "1,*,*") 16678 (set_attr "prefix_data16" "1,*,*")
15324 (set_attr "prefix_extra" "1") 16679 (set_attr "prefix_extra" "1")
15325 (set_attr "prefix" "orig,maybe_evex,evex") 16680 (set_attr "prefix" "orig,maybe_evex,evex")
15326 (set_attr "mode" "<sseinsnmode>")]) 16681 (set_attr "mode" "<sseinsnmode>")])
15327 16682
15328 (define_insn "*ssse3_pmulhrswv4hi3" 16683 (define_expand "smulhrsv4hi3"
15329 [(set (match_operand:V4HI 0 "register_operand" "=y") 16684 [(set (match_operand:V4HI 0 "register_operand")
15330 (truncate:V4HI 16685 (truncate:V4HI
15331 (lshiftrt:V4SI 16686 (lshiftrt:V4SI
15332 (plus:V4SI 16687 (plus:V4SI
15333 (lshiftrt:V4SI 16688 (lshiftrt:V4SI
15334 (mult:V4SI 16689 (mult:V4SI
15335 (sign_extend:V4SI 16690 (sign_extend:V4SI
15336 (match_operand:V4HI 1 "nonimmediate_operand" "%0")) 16691 (match_operand:V4HI 1 "register_operand"))
15337 (sign_extend:V4SI 16692 (sign_extend:V4SI
15338 (match_operand:V4HI 2 "nonimmediate_operand" "ym"))) 16693 (match_operand:V4HI 2 "register_operand")))
16694 (const_int 14))
16695 (match_dup 3))
16696 (const_int 1))))]
16697 "TARGET_MMX_WITH_SSE && TARGET_SSSE3"
16698 {
16699 operands[3] = CONST1_RTX(V4HImode);
16700 ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);
16701 })
16702
16703 (define_expand "ssse3_pmulhrswv4hi3"
16704 [(set (match_operand:V4HI 0 "register_operand")
16705 (truncate:V4HI
16706 (lshiftrt:V4SI
16707 (plus:V4SI
16708 (lshiftrt:V4SI
16709 (mult:V4SI
16710 (sign_extend:V4SI
16711 (match_operand:V4HI 1 "register_mmxmem_operand"))
16712 (sign_extend:V4SI
16713 (match_operand:V4HI 2 "register_mmxmem_operand")))
16714 (const_int 14))
16715 (match_dup 3))
16716 (const_int 1))))]
16717 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
16718 {
16719 operands[3] = CONST1_RTX(V4HImode);
16720 ix86_fixup_binary_operands_no_copy (MULT, V4HImode, operands);
16721 })
16722
16723 (define_insn "*ssse3_pmulhrswv4hi3"
16724 [(set (match_operand:V4HI 0 "register_operand" "=y,x,Yv")
16725 (truncate:V4HI
16726 (lshiftrt:V4SI
16727 (plus:V4SI
16728 (lshiftrt:V4SI
16729 (mult:V4SI
16730 (sign_extend:V4SI
16731 (match_operand:V4HI 1 "register_mmxmem_operand" "%0,0,Yv"))
16732 (sign_extend:V4SI
16733 (match_operand:V4HI 2 "register_mmxmem_operand" "ym,x,Yv")))
15339 (const_int 14)) 16734 (const_int 14))
15340 (match_operand:V4HI 3 "const1_operand")) 16735 (match_operand:V4HI 3 "const1_operand"))
15341 (const_int 1))))] 16736 (const_int 1))))]
15342 "TARGET_SSSE3 && !(MEM_P (operands[1]) && MEM_P (operands[2]))" 16737 "(TARGET_MMX || TARGET_MMX_WITH_SSE)
15343 "pmulhrsw\t{%2, %0|%0, %2}" 16738 && TARGET_SSSE3
15344 [(set_attr "type" "sseimul") 16739 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
16740 "@
16741 pmulhrsw\t{%2, %0|%0, %2}
16742 pmulhrsw\t{%2, %0|%0, %2}
16743 vpmulhrsw\t{%2, %1, %0|%0, %1, %2}"
16744 [(set_attr "isa" "*,noavx,avx")
16745 (set_attr "mmx_isa" "native,*,*")
16746 (set_attr "type" "sseimul")
15345 (set_attr "prefix_extra" "1") 16747 (set_attr "prefix_extra" "1")
15346 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16748 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15347 (set_attr "mode" "DI")]) 16749 (set_attr "mode" "DI,TI,TI")])
15348 16750
15349 (define_insn "<ssse3_avx2>_pshufb<mode>3<mask_name>" 16751 (define_insn "<ssse3_avx2>_pshufb<mode>3<mask_name>"
15350 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v") 16752 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v")
15351 (unspec:VI1_AVX512 16753 (unspec:VI1_AVX512
15352 [(match_operand:VI1_AVX512 1 "register_operand" "0,x,v") 16754 [(match_operand:VI1_AVX512 1 "register_operand" "0,x,v")
15363 (set_attr "prefix_extra" "1") 16765 (set_attr "prefix_extra" "1")
15364 (set_attr "prefix" "orig,maybe_evex,evex") 16766 (set_attr "prefix" "orig,maybe_evex,evex")
15365 (set_attr "btver2_decode" "vector") 16767 (set_attr "btver2_decode" "vector")
15366 (set_attr "mode" "<sseinsnmode>")]) 16768 (set_attr "mode" "<sseinsnmode>")])
15367 16769
15368 (define_insn "ssse3_pshufbv8qi3" 16770 (define_insn_and_split "ssse3_pshufbv8qi3"
15369 [(set (match_operand:V8QI 0 "register_operand" "=y") 16771 [(set (match_operand:V8QI 0 "register_operand" "=y,x,Yv")
15370 (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "0") 16772 (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "0,0,Yv")
15371 (match_operand:V8QI 2 "nonimmediate_operand" "ym")] 16773 (match_operand:V8QI 2 "register_mmxmem_operand" "ym,x,Yv")]
15372 UNSPEC_PSHUFB))] 16774 UNSPEC_PSHUFB))
15373 "TARGET_SSSE3" 16775 (clobber (match_scratch:V4SI 3 "=X,x,Yv"))]
15374 "pshufb\t{%2, %0|%0, %2}"; 16776 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15375 [(set_attr "type" "sselog1") 16777 "@
16778 pshufb\t{%2, %0|%0, %2}
16779 #
16780 #"
16781 "TARGET_SSSE3 && reload_completed
16782 && SSE_REGNO_P (REGNO (operands[0]))"
16783 [(set (match_dup 3) (match_dup 5))
16784 (set (match_dup 3)
16785 (and:V4SI (match_dup 3) (match_dup 2)))
16786 (set (match_dup 0)
16787 (unspec:V16QI [(match_dup 1) (match_dup 4)] UNSPEC_PSHUFB))]
16788 {
16789 /* Emulate MMX version of pshufb with SSE version by masking out the
16790 bit 3 of the shuffle control byte. */
16791 operands[0] = lowpart_subreg (V16QImode, operands[0],
16792 GET_MODE (operands[0]));
16793 operands[1] = lowpart_subreg (V16QImode, operands[1],
16794 GET_MODE (operands[1]));
16795 operands[2] = lowpart_subreg (V4SImode, operands[2],
16796 GET_MODE (operands[2]));
16797 operands[4] = lowpart_subreg (V16QImode, operands[3],
16798 GET_MODE (operands[3]));
16799 rtvec par = gen_rtvec (4, GEN_INT (0xf7f7f7f7),
16800 GEN_INT (0xf7f7f7f7),
16801 GEN_INT (0xf7f7f7f7),
16802 GEN_INT (0xf7f7f7f7));
16803 rtx vec_const = gen_rtx_CONST_VECTOR (V4SImode, par);
16804 operands[5] = force_const_mem (V4SImode, vec_const);
16805 }
16806 [(set_attr "mmx_isa" "native,sse_noavx,avx")
15376 (set_attr "prefix_extra" "1") 16807 (set_attr "prefix_extra" "1")
15377 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16808 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15378 (set_attr "mode" "DI")]) 16809 (set_attr "mode" "DI,TI,TI")])
15379 16810
15380 (define_insn "<ssse3_avx2>_psign<mode>3" 16811 (define_insn "<ssse3_avx2>_psign<mode>3"
15381 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x,x") 16812 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x,x")
15382 (unspec:VI124_AVX2 16813 (unspec:VI124_AVX2
15383 [(match_operand:VI124_AVX2 1 "register_operand" "0,x") 16814 [(match_operand:VI124_AVX2 1 "register_operand" "0,x")
15393 (set_attr "prefix_extra" "1") 16824 (set_attr "prefix_extra" "1")
15394 (set_attr "prefix" "orig,vex") 16825 (set_attr "prefix" "orig,vex")
15395 (set_attr "mode" "<sseinsnmode>")]) 16826 (set_attr "mode" "<sseinsnmode>")])
15396 16827
15397 (define_insn "ssse3_psign<mode>3" 16828 (define_insn "ssse3_psign<mode>3"
15398 [(set (match_operand:MMXMODEI 0 "register_operand" "=y") 16829 [(set (match_operand:MMXMODEI 0 "register_operand" "=y,x,Yv")
15399 (unspec:MMXMODEI 16830 (unspec:MMXMODEI
15400 [(match_operand:MMXMODEI 1 "register_operand" "0") 16831 [(match_operand:MMXMODEI 1 "register_operand" "0,0,Yv")
15401 (match_operand:MMXMODEI 2 "nonimmediate_operand" "ym")] 16832 (match_operand:MMXMODEI 2 "register_mmxmem_operand" "ym,x,Yv")]
15402 UNSPEC_PSIGN))] 16833 UNSPEC_PSIGN))]
15403 "TARGET_SSSE3" 16834 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15404 "psign<mmxvecsize>\t{%2, %0|%0, %2}"; 16835 "@
15405 [(set_attr "type" "sselog1") 16836 psign<mmxvecsize>\t{%2, %0|%0, %2}
16837 psign<mmxvecsize>\t{%2, %0|%0, %2}
16838 vpsign<mmxvecsize>\t{%2, %1, %0|%0, %1, %2}"
16839 [(set_attr "isa" "*,noavx,avx")
16840 (set_attr "mmx_isa" "native,*,*")
16841 (set_attr "type" "sselog1")
15406 (set_attr "prefix_extra" "1") 16842 (set_attr "prefix_extra" "1")
15407 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16843 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15408 (set_attr "mode" "DI")]) 16844 (set_attr "mode" "DI,TI,TI")])
15409 16845
15410 (define_insn "<ssse3_avx2>_palignr<mode>_mask" 16846 (define_insn "<ssse3_avx2>_palignr<mode>_mask"
15411 [(set (match_operand:VI1_AVX512 0 "register_operand" "=v") 16847 [(set (match_operand:VI1_AVX512 0 "register_operand" "=v")
15412 (vec_merge:VI1_AVX512 16848 (vec_merge:VI1_AVX512
15413 (unspec:VI1_AVX512 16849 (unspec:VI1_AVX512
15458 (set_attr "prefix_extra" "1") 16894 (set_attr "prefix_extra" "1")
15459 (set_attr "length_immediate" "1") 16895 (set_attr "length_immediate" "1")
15460 (set_attr "prefix" "orig,vex,evex") 16896 (set_attr "prefix" "orig,vex,evex")
15461 (set_attr "mode" "<sseinsnmode>")]) 16897 (set_attr "mode" "<sseinsnmode>")])
15462 16898
15463 (define_insn "ssse3_palignrdi" 16899 (define_insn_and_split "ssse3_palignrdi"
15464 [(set (match_operand:DI 0 "register_operand" "=y") 16900 [(set (match_operand:DI 0 "register_operand" "=y,x,Yv")
15465 (unspec:DI [(match_operand:DI 1 "register_operand" "0") 16901 (unspec:DI [(match_operand:DI 1 "register_operand" "0,0,Yv")
15466 (match_operand:DI 2 "nonimmediate_operand" "ym") 16902 (match_operand:DI 2 "register_mmxmem_operand" "ym,x,Yv")
15467 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")] 16903 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n,n,n")]
15468 UNSPEC_PALIGNR))] 16904 UNSPEC_PALIGNR))]
15469 "TARGET_SSSE3" 16905 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15470 { 16906 {
15471 operands[3] = GEN_INT (INTVAL (operands[3]) / 8); 16907 switch (which_alternative)
15472 return "palignr\t{%3, %2, %0|%0, %2, %3}"; 16908 {
16909 case 0:
16910 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
16911 return "palignr\t{%3, %2, %0|%0, %2, %3}";
16912 case 1:
16913 case 2:
16914 return "#";
16915 default:
16916 gcc_unreachable ();
16917 }
15473 } 16918 }
15474 [(set_attr "type" "sseishft") 16919 "TARGET_SSSE3 && reload_completed
16920 && SSE_REGNO_P (REGNO (operands[0]))"
16921 [(set (match_dup 0)
16922 (lshiftrt:V1TI (match_dup 0) (match_dup 3)))]
16923 {
16924 /* Emulate MMX palignrdi with SSE psrldq. */
16925 rtx op0 = lowpart_subreg (V2DImode, operands[0],
16926 GET_MODE (operands[0]));
16927 if (TARGET_AVX)
16928 emit_insn (gen_vec_concatv2di (op0, operands[2], operands[1]));
16929 else
16930 {
16931 /* NB: SSE can only concatenate OP0 and OP1 to OP0. */
16932 emit_insn (gen_vec_concatv2di (op0, operands[1], operands[2]));
16933 /* Swap bits 0:63 with bits 64:127. */
16934 rtx mask = gen_rtx_PARALLEL (VOIDmode,
16935 gen_rtvec (4, GEN_INT (2),
16936 GEN_INT (3),
16937 GEN_INT (0),
16938 GEN_INT (1)));
16939 rtx op1 = lowpart_subreg (V4SImode, op0, GET_MODE (op0));
16940 rtx op2 = gen_rtx_VEC_SELECT (V4SImode, op1, mask);
16941 emit_insn (gen_rtx_SET (op1, op2));
16942 }
16943 operands[0] = lowpart_subreg (V1TImode, op0, GET_MODE (op0));
16944 }
16945 [(set_attr "mmx_isa" "native,sse_noavx,avx")
16946 (set_attr "type" "sseishft")
15475 (set_attr "atom_unit" "sishuf") 16947 (set_attr "atom_unit" "sishuf")
15476 (set_attr "prefix_extra" "1") 16948 (set_attr "prefix_extra" "1")
15477 (set_attr "length_immediate" "1") 16949 (set_attr "length_immediate" "1")
15478 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 16950 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15479 (set_attr "mode" "DI")]) 16951 (set_attr "mode" "DI,TI,TI")])
15480 16952
15481 ;; Mode iterator to handle singularity w/ absence of V2DI and V4DI 16953 ;; Mode iterator to handle singularity w/ absence of V2DI and V4DI
15482 ;; modes for abs instruction on pre AVX-512 targets. 16954 ;; modes for abs instruction on pre AVX-512 targets.
15483 (define_mode_iterator VI1248_AVX512VL_AVX512BW 16955 (define_mode_iterator VI1248_AVX512VL_AVX512BW
15484 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI 16956 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
15537 ix86_expand_sse2_abs (operands[0], operands[1]); 17009 ix86_expand_sse2_abs (operands[0], operands[1]);
15538 DONE; 17010 DONE;
15539 } 17011 }
15540 }) 17012 })
15541 17013
15542 (define_insn "abs<mode>2" 17014 (define_insn "ssse3_abs<mode>2"
15543 [(set (match_operand:MMXMODEI 0 "register_operand" "=y") 17015 [(set (match_operand:MMXMODEI 0 "register_operand" "=y,Yv")
15544 (abs:MMXMODEI 17016 (abs:MMXMODEI
15545 (match_operand:MMXMODEI 1 "nonimmediate_operand" "ym")))] 17017 (match_operand:MMXMODEI 1 "register_mmxmem_operand" "ym,Yv")))]
15546 "TARGET_SSSE3" 17018 "(TARGET_MMX || TARGET_MMX_WITH_SSE) && TARGET_SSSE3"
15547 "pabs<mmxvecsize>\t{%1, %0|%0, %1}"; 17019 "@
15548 [(set_attr "type" "sselog1") 17020 pabs<mmxvecsize>\t{%1, %0|%0, %1}
17021 %vpabs<mmxvecsize>\t{%1, %0|%0, %1}"
17022 [(set_attr "mmx_isa" "native,*")
17023 (set_attr "type" "sselog1")
15549 (set_attr "prefix_rep" "0") 17024 (set_attr "prefix_rep" "0")
15550 (set_attr "prefix_extra" "1") 17025 (set_attr "prefix_extra" "1")
15551 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)")) 17026 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15552 (set_attr "mode" "DI")]) 17027 (set_attr "mode" "DI,TI")])
17028
17029 (define_insn "abs<mode>2"
17030 [(set (match_operand:MMXMODEI 0 "register_operand")
17031 (abs:MMXMODEI
17032 (match_operand:MMXMODEI 1 "register_operand")))]
17033 "TARGET_MMX_WITH_SSE && TARGET_SSSE3")
15553 17034
15554 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 17035 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15555 ;; 17036 ;;
15556 ;; AMD SSE4A instructions 17037 ;; AMD SSE4A instructions
15557 ;; 17038 ;;
15677 (set_attr "prefix_data16" "1,1,*") 17158 (set_attr "prefix_data16" "1,1,*")
15678 (set_attr "prefix_extra" "1") 17159 (set_attr "prefix_extra" "1")
15679 (set_attr "prefix" "orig,orig,vex") 17160 (set_attr "prefix" "orig,orig,vex")
15680 (set_attr "btver2_decode" "vector,vector,vector") 17161 (set_attr "btver2_decode" "vector,vector,vector")
15681 (set_attr "mode" "<MODE>")]) 17162 (set_attr "mode" "<MODE>")])
17163
17164 ;; Also define scalar versions. These are used for conditional move.
17165 ;; Using subregs into vector modes causes register allocation lossage.
17166 ;; These patterns do not allow memory operands because the native
17167 ;; instructions read the full 128-bits.
17168
17169 (define_insn "sse4_1_blendv<ssemodesuffix>"
17170 [(set (match_operand:MODEF 0 "register_operand" "=Yr,*x,x")
17171 (unspec:MODEF
17172 [(match_operand:MODEF 1 "register_operand" "0,0,x")
17173 (match_operand:MODEF 2 "register_operand" "Yr,*x,x")
17174 (match_operand:MODEF 3 "register_operand" "Yz,Yz,x")]
17175 UNSPEC_BLENDV))]
17176 "TARGET_SSE4_1"
17177 {
17178 if (get_attr_mode (insn) == MODE_V4SF)
17179 return (which_alternative == 2
17180 ? "vblendvps\t{%3, %2, %1, %0|%0, %1, %2, %3}"
17181 : "blendvps\t{%3, %2, %0|%0, %2, %3}");
17182 else
17183 return (which_alternative == 2
17184 ? "vblendv<ssevecmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
17185 : "blendv<ssevecmodesuffix>\t{%3, %2, %0|%0, %2, %3}");
17186 }
17187 [(set_attr "isa" "noavx,noavx,avx")
17188 (set_attr "type" "ssemov")
17189 (set_attr "length_immediate" "1")
17190 (set_attr "prefix_data16" "1,1,*")
17191 (set_attr "prefix_extra" "1")
17192 (set_attr "prefix" "orig,orig,vex")
17193 (set_attr "btver2_decode" "vector,vector,vector")
17194 (set (attr "mode")
17195 (cond [(match_test "TARGET_AVX")
17196 (const_string "<ssevecmode>")
17197 (match_test "optimize_function_for_size_p (cfun)")
17198 (const_string "V4SF")
17199 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
17200 (const_string "V4SF")
17201 ]
17202 (const_string "<ssevecmode>")))])
17203
17204 (define_insn_and_split "*<sse4_1>_blendv<ssemodesuffix><avxsizesuffix>_lt"
17205 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
17206 (unspec:VF_128_256
17207 [(match_operand:VF_128_256 1 "register_operand" "0,0,x")
17208 (match_operand:VF_128_256 2 "vector_operand" "YrBm,*xBm,xm")
17209 (lt:VF_128_256
17210 (match_operand:<sseintvecmode> 3 "register_operand" "Yz,Yz,x")
17211 (match_operand:<sseintvecmode> 4 "const0_operand" "C,C,C"))]
17212 UNSPEC_BLENDV))]
17213 "TARGET_SSE4_1"
17214 "#"
17215 "&& reload_completed"
17216 [(set (match_dup 0)
17217 (unspec:VF_128_256
17218 [(match_dup 1) (match_dup 2) (match_dup 3)] UNSPEC_BLENDV))]
17219 "operands[3] = gen_lowpart (<MODE>mode, operands[3]);"
17220 [(set_attr "isa" "noavx,noavx,avx")
17221 (set_attr "type" "ssemov")
17222 (set_attr "length_immediate" "1")
17223 (set_attr "prefix_data16" "1,1,*")
17224 (set_attr "prefix_extra" "1")
17225 (set_attr "prefix" "orig,orig,vex")
17226 (set_attr "btver2_decode" "vector,vector,vector")
17227 (set_attr "mode" "<MODE>")])
17228
17229 (define_mode_attr ssefltmodesuffix
17230 [(V2DI "pd") (V4DI "pd") (V4SI "ps") (V8SI "ps")])
17231
17232 (define_mode_attr ssefltvecmode
17233 [(V2DI "V2DF") (V4DI "V4DF") (V4SI "V4SF") (V8SI "V8SF")])
17234
17235 (define_insn_and_split "*<sse4_1>_blendv<ssefltmodesuffix><avxsizesuffix>_ltint"
17236 [(set (match_operand:<ssebytemode> 0 "register_operand" "=Yr,*x,x")
17237 (unspec:<ssebytemode>
17238 [(match_operand:<ssebytemode> 1 "register_operand" "0,0,x")
17239 (match_operand:<ssebytemode> 2 "vector_operand" "YrBm,*xBm,xm")
17240 (subreg:<ssebytemode>
17241 (lt:VI48_AVX
17242 (match_operand:VI48_AVX 3 "register_operand" "Yz,Yz,x")
17243 (match_operand:VI48_AVX 4 "const0_operand" "C,C,C")) 0)]
17244 UNSPEC_BLENDV))]
17245 "TARGET_SSE4_1"
17246 "#"
17247 "&& reload_completed"
17248 [(set (match_dup 0)
17249 (unspec:<ssefltvecmode>
17250 [(match_dup 1) (match_dup 2) (match_dup 3)] UNSPEC_BLENDV))]
17251 {
17252 operands[0] = gen_lowpart (<ssefltvecmode>mode, operands[0]);
17253 operands[1] = gen_lowpart (<ssefltvecmode>mode, operands[1]);
17254 operands[2] = gen_lowpart (<ssefltvecmode>mode, operands[2]);
17255 operands[3] = gen_lowpart (<ssefltvecmode>mode, operands[3]);
17256 }
17257 [(set_attr "isa" "noavx,noavx,avx")
17258 (set_attr "type" "ssemov")
17259 (set_attr "length_immediate" "1")
17260 (set_attr "prefix_data16" "1,1,*")
17261 (set_attr "prefix_extra" "1")
17262 (set_attr "prefix" "orig,orig,vex")
17263 (set_attr "btver2_decode" "vector,vector,vector")
17264 (set_attr "mode" "<ssefltvecmode>")])
15682 17265
15683 (define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>" 17266 (define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>"
15684 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x") 17267 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
15685 (unspec:VF_128_256 17268 (unspec:VF_128_256
15686 [(match_operand:VF_128_256 1 "vector_operand" "%0,0,x") 17269 [(match_operand:VF_128_256 1 "vector_operand" "%0,0,x")
15776 (set_attr "length_immediate" "*,*,1") 17359 (set_attr "length_immediate" "*,*,1")
15777 (set_attr "prefix" "orig,orig,vex") 17360 (set_attr "prefix" "orig,orig,vex")
15778 (set_attr "btver2_decode" "vector,vector,vector") 17361 (set_attr "btver2_decode" "vector,vector,vector")
15779 (set_attr "mode" "<sseinsnmode>")]) 17362 (set_attr "mode" "<sseinsnmode>")])
15780 17363
17364 (define_insn_and_split "*<sse4_1_avx2>_pblendvb_lt"
17365 [(set (match_operand:VI1_AVX2 0 "register_operand" "=Yr,*x,x")
17366 (unspec:VI1_AVX2
17367 [(match_operand:VI1_AVX2 1 "register_operand" "0,0,x")
17368 (match_operand:VI1_AVX2 2 "vector_operand" "YrBm,*xBm,xm")
17369 (lt:VI1_AVX2 (match_operand:VI1_AVX2 3 "register_operand" "Yz,Yz,x")
17370 (match_operand:VI1_AVX2 4 "const0_operand" "C,C,C"))]
17371 UNSPEC_BLENDV))]
17372 "TARGET_SSE4_1"
17373 "#"
17374 ""
17375 [(set (match_dup 0)
17376 (unspec:VI1_AVX2
17377 [(match_dup 1) (match_dup 2) (match_dup 3)] UNSPEC_BLENDV))]
17378 ""
17379 [(set_attr "isa" "noavx,noavx,avx")
17380 (set_attr "type" "ssemov")
17381 (set_attr "prefix_extra" "1")
17382 (set_attr "length_immediate" "*,*,1")
17383 (set_attr "prefix" "orig,orig,vex")
17384 (set_attr "btver2_decode" "vector,vector,vector")
17385 (set_attr "mode" "<sseinsnmode>")])
17386
15781 (define_insn "sse4_1_pblendw" 17387 (define_insn "sse4_1_pblendw"
15782 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,x") 17388 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,x")
15783 (vec_merge:V8HI 17389 (vec_merge:V8HI
15784 (match_operand:V8HI 2 "vector_operand" "YrBm,*xBm,xm") 17390 (match_operand:V8HI 2 "vector_operand" "YrBm,*xBm,xm")
15785 (match_operand:V8HI 1 "register_operand" "0,0,x") 17391 (match_operand:V8HI 1 "register_operand" "0,0,x")
15876 17482
15877 (define_insn "sse4_1_<code>v8qiv8hi2<mask_name>" 17483 (define_insn "sse4_1_<code>v8qiv8hi2<mask_name>"
15878 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,v") 17484 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,v")
15879 (any_extend:V8HI 17485 (any_extend:V8HI
15880 (vec_select:V8QI 17486 (vec_select:V8QI
15881 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17487 (match_operand:V16QI 1 "register_operand" "Yr,*x,v")
15882 (parallel [(const_int 0) (const_int 1) 17488 (parallel [(const_int 0) (const_int 1)
15883 (const_int 2) (const_int 3) 17489 (const_int 2) (const_int 3)
15884 (const_int 4) (const_int 5) 17490 (const_int 4) (const_int 5)
15885 (const_int 6) (const_int 7)]))))] 17491 (const_int 6) (const_int 7)]))))]
15886 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>" 17492 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
15887 "%vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}" 17493 "%vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15888 [(set_attr "isa" "noavx,noavx,avx") 17494 [(set_attr "isa" "noavx,noavx,avx")
15889 (set_attr "type" "ssemov") 17495 (set_attr "type" "ssemov")
15890 (set_attr "prefix_extra" "1") 17496 (set_attr "prefix_extra" "1")
15891 (set_attr "prefix" "orig,orig,maybe_evex") 17497 (set_attr "prefix" "orig,orig,maybe_evex")
15892 (set_attr "mode" "TI")]) 17498 (set_attr "mode" "TI")])
17499
17500 (define_insn "*sse4_1_<code>v8qiv8hi2<mask_name>_1"
17501 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,v")
17502 (any_extend:V8HI
17503 (match_operand:V8QI 1 "memory_operand" "m,m,m")))]
17504 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
17505 "%vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17506 [(set_attr "isa" "noavx,noavx,avx")
17507 (set_attr "type" "ssemov")
17508 (set_attr "prefix_extra" "1")
17509 (set_attr "prefix" "orig,orig,maybe_evex")
17510 (set_attr "mode" "TI")])
17511
17512 (define_insn_and_split "*sse4_1_<code>v8qiv8hi2<mask_name>_2"
17513 [(set (match_operand:V8HI 0 "register_operand")
17514 (any_extend:V8HI
17515 (vec_select:V8QI
17516 (subreg:V16QI
17517 (vec_concat:V2DI
17518 (match_operand:DI 1 "memory_operand")
17519 (const_int 0)) 0)
17520 (parallel [(const_int 0) (const_int 1)
17521 (const_int 2) (const_int 3)
17522 (const_int 4) (const_int 5)
17523 (const_int 6) (const_int 7)]))))]
17524 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>
17525 && ix86_pre_reload_split ()"
17526 "#"
17527 "&& 1"
17528 [(set (match_dup 0)
17529 (any_extend:V8HI (match_dup 1)))]
17530 "operands[1] = adjust_address_nv (operands[1], V8QImode, 0);")
15893 17531
15894 (define_insn "<mask_codefor>avx512f_<code>v16qiv16si2<mask_name>" 17532 (define_insn "<mask_codefor>avx512f_<code>v16qiv16si2<mask_name>"
15895 [(set (match_operand:V16SI 0 "register_operand" "=v") 17533 [(set (match_operand:V16SI 0 "register_operand" "=v")
15896 (any_extend:V16SI 17534 (any_extend:V16SI
15897 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))] 17535 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
15903 17541
15904 (define_insn "avx2_<code>v8qiv8si2<mask_name>" 17542 (define_insn "avx2_<code>v8qiv8si2<mask_name>"
15905 [(set (match_operand:V8SI 0 "register_operand" "=v") 17543 [(set (match_operand:V8SI 0 "register_operand" "=v")
15906 (any_extend:V8SI 17544 (any_extend:V8SI
15907 (vec_select:V8QI 17545 (vec_select:V8QI
15908 (match_operand:V16QI 1 "nonimmediate_operand" "vm") 17546 (match_operand:V16QI 1 "register_operand" "v")
15909 (parallel [(const_int 0) (const_int 1) 17547 (parallel [(const_int 0) (const_int 1)
15910 (const_int 2) (const_int 3) 17548 (const_int 2) (const_int 3)
15911 (const_int 4) (const_int 5) 17549 (const_int 4) (const_int 5)
15912 (const_int 6) (const_int 7)]))))] 17550 (const_int 6) (const_int 7)]))))]
15913 "TARGET_AVX2 && <mask_avx512vl_condition>" 17551 "TARGET_AVX2 && <mask_avx512vl_condition>"
15914 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}" 17552 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15915 [(set_attr "type" "ssemov") 17553 [(set_attr "type" "ssemov")
15916 (set_attr "prefix_extra" "1") 17554 (set_attr "prefix_extra" "1")
15917 (set_attr "prefix" "maybe_evex") 17555 (set_attr "prefix" "maybe_evex")
15918 (set_attr "mode" "OI")]) 17556 (set_attr "mode" "OI")])
15919 17557
17558 (define_insn "*avx2_<code>v8qiv8si2<mask_name>_1"
17559 [(set (match_operand:V8SI 0 "register_operand" "=v")
17560 (any_extend:V8SI
17561 (match_operand:V8QI 1 "memory_operand" "m")))]
17562 "TARGET_AVX2 && <mask_avx512vl_condition>"
17563 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17564 [(set_attr "type" "ssemov")
17565 (set_attr "prefix_extra" "1")
17566 (set_attr "prefix" "maybe_evex")
17567 (set_attr "mode" "OI")])
17568
17569 (define_insn_and_split "*avx2_<code>v8qiv8si2<mask_name>_2"
17570 [(set (match_operand:V8SI 0 "register_operand")
17571 (any_extend:V8SI
17572 (vec_select:V8QI
17573 (subreg:V16QI
17574 (vec_concat:V2DI
17575 (match_operand:DI 1 "memory_operand")
17576 (const_int 0)) 0)
17577 (parallel [(const_int 0) (const_int 1)
17578 (const_int 2) (const_int 3)
17579 (const_int 4) (const_int 5)
17580 (const_int 6) (const_int 7)]))))]
17581 "TARGET_AVX2 && <mask_avx512vl_condition>
17582 && ix86_pre_reload_split ()"
17583 "#"
17584 "&& 1"
17585 [(set (match_dup 0)
17586 (any_extend:V8SI (match_dup 1)))]
17587 "operands[1] = adjust_address_nv (operands[1], V8QImode, 0);")
17588
15920 (define_insn "sse4_1_<code>v4qiv4si2<mask_name>" 17589 (define_insn "sse4_1_<code>v4qiv4si2<mask_name>"
15921 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v") 17590 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
15922 (any_extend:V4SI 17591 (any_extend:V4SI
15923 (vec_select:V4QI 17592 (vec_select:V4QI
15924 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17593 (match_operand:V16QI 1 "register_operand" "Yr,*x,v")
15925 (parallel [(const_int 0) (const_int 1) 17594 (parallel [(const_int 0) (const_int 1)
15926 (const_int 2) (const_int 3)]))))] 17595 (const_int 2) (const_int 3)]))))]
15927 "TARGET_SSE4_1 && <mask_avx512vl_condition>" 17596 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15928 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}" 17597 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15929 [(set_attr "isa" "noavx,noavx,avx") 17598 [(set_attr "isa" "noavx,noavx,avx")
15930 (set_attr "type" "ssemov") 17599 (set_attr "type" "ssemov")
15931 (set_attr "prefix_extra" "1") 17600 (set_attr "prefix_extra" "1")
15932 (set_attr "prefix" "orig,orig,maybe_evex") 17601 (set_attr "prefix" "orig,orig,maybe_evex")
15933 (set_attr "mode" "TI")]) 17602 (set_attr "mode" "TI")])
17603
17604 (define_insn "*sse4_1_<code>v4qiv4si2<mask_name>_1"
17605 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
17606 (any_extend:V4SI
17607 (match_operand:V4QI 1 "memory_operand" "m,m,m")))]
17608 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
17609 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17610 [(set_attr "isa" "noavx,noavx,avx")
17611 (set_attr "type" "ssemov")
17612 (set_attr "prefix_extra" "1")
17613 (set_attr "prefix" "orig,orig,maybe_evex")
17614 (set_attr "mode" "TI")])
17615
17616 (define_insn_and_split "*sse4_1_<code>v4qiv4si2<mask_name>_2"
17617 [(set (match_operand:V4SI 0 "register_operand")
17618 (any_extend:V4SI
17619 (vec_select:V4QI
17620 (subreg:V16QI
17621 (vec_merge:V4SI
17622 (vec_duplicate:V4SI
17623 (match_operand:SI 1 "memory_operand"))
17624 (const_vector:V4SI
17625 [(const_int 0) (const_int 0)
17626 (const_int 0) (const_int 0)])
17627 (const_int 1)) 0)
17628 (parallel [(const_int 0) (const_int 1)
17629 (const_int 2) (const_int 3)]))))]
17630 "TARGET_SSE4_1 && <mask_avx512vl_condition>
17631 && ix86_pre_reload_split ()"
17632 "#"
17633 "&& 1"
17634 [(set (match_dup 0)
17635 (any_extend:V4SI (match_dup 1)))]
17636 "operands[1] = adjust_address_nv (operands[1], V4QImode, 0);")
15934 17637
15935 (define_insn "avx512f_<code>v16hiv16si2<mask_name>" 17638 (define_insn "avx512f_<code>v16hiv16si2<mask_name>"
15936 [(set (match_operand:V16SI 0 "register_operand" "=v") 17639 [(set (match_operand:V16SI 0 "register_operand" "=v")
15937 (any_extend:V16SI 17640 (any_extend:V16SI
15938 (match_operand:V16HI 1 "nonimmediate_operand" "vm")))] 17641 (match_operand:V16HI 1 "nonimmediate_operand" "vm")))]
15955 17658
15956 (define_insn "sse4_1_<code>v4hiv4si2<mask_name>" 17659 (define_insn "sse4_1_<code>v4hiv4si2<mask_name>"
15957 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v") 17660 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
15958 (any_extend:V4SI 17661 (any_extend:V4SI
15959 (vec_select:V4HI 17662 (vec_select:V4HI
15960 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17663 (match_operand:V8HI 1 "register_operand" "Yr,*x,v")
15961 (parallel [(const_int 0) (const_int 1) 17664 (parallel [(const_int 0) (const_int 1)
15962 (const_int 2) (const_int 3)]))))] 17665 (const_int 2) (const_int 3)]))))]
15963 "TARGET_SSE4_1 && <mask_avx512vl_condition>" 17666 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15964 "%vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}" 17667 "%vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15965 [(set_attr "isa" "noavx,noavx,avx") 17668 [(set_attr "isa" "noavx,noavx,avx")
15966 (set_attr "type" "ssemov") 17669 (set_attr "type" "ssemov")
15967 (set_attr "prefix_extra" "1") 17670 (set_attr "prefix_extra" "1")
15968 (set_attr "prefix" "orig,orig,maybe_evex") 17671 (set_attr "prefix" "orig,orig,maybe_evex")
15969 (set_attr "mode" "TI")]) 17672 (set_attr "mode" "TI")])
15970 17673
17674 (define_insn "*sse4_1_<code>v4hiv4si2<mask_name>_1"
17675 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
17676 (any_extend:V4SI
17677 (match_operand:V4HI 1 "memory_operand" "m,m,m")))]
17678 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
17679 "%vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17680 [(set_attr "isa" "noavx,noavx,avx")
17681 (set_attr "type" "ssemov")
17682 (set_attr "prefix_extra" "1")
17683 (set_attr "prefix" "orig,orig,maybe_evex")
17684 (set_attr "mode" "TI")])
17685
17686 (define_insn_and_split "*sse4_1_<code>v4hiv4si2<mask_name>_2"
17687 [(set (match_operand:V4SI 0 "register_operand")
17688 (any_extend:V4SI
17689 (vec_select:V4HI
17690 (subreg:V8HI
17691 (vec_concat:V2DI
17692 (match_operand:DI 1 "memory_operand")
17693 (const_int 0)) 0)
17694 (parallel [(const_int 0) (const_int 1)
17695 (const_int 2) (const_int 3)]))))]
17696 "TARGET_SSE4_1 && <mask_avx512vl_condition>
17697 && ix86_pre_reload_split ()"
17698 "#"
17699 "&& 1"
17700 [(set (match_dup 0)
17701 (any_extend:V4SI (match_dup 1)))]
17702 "operands[1] = adjust_address_nv (operands[1], V4HImode, 0);")
17703
15971 (define_insn "avx512f_<code>v8qiv8di2<mask_name>" 17704 (define_insn "avx512f_<code>v8qiv8di2<mask_name>"
15972 [(set (match_operand:V8DI 0 "register_operand" "=v") 17705 [(set (match_operand:V8DI 0 "register_operand" "=v")
15973 (any_extend:V8DI 17706 (any_extend:V8DI
15974 (vec_select:V8QI 17707 (vec_select:V8QI
15975 (match_operand:V16QI 1 "nonimmediate_operand" "vm") 17708 (match_operand:V16QI 1 "register_operand" "v")
15976 (parallel [(const_int 0) (const_int 1) 17709 (parallel [(const_int 0) (const_int 1)
15977 (const_int 2) (const_int 3) 17710 (const_int 2) (const_int 3)
15978 (const_int 4) (const_int 5) 17711 (const_int 4) (const_int 5)
15979 (const_int 6) (const_int 7)]))))] 17712 (const_int 6) (const_int 7)]))))]
15980 "TARGET_AVX512F" 17713 "TARGET_AVX512F"
15981 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}" 17714 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15982 [(set_attr "type" "ssemov") 17715 [(set_attr "type" "ssemov")
15983 (set_attr "prefix" "evex") 17716 (set_attr "prefix" "evex")
15984 (set_attr "mode" "XI")]) 17717 (set_attr "mode" "XI")])
17718
17719 (define_insn "*avx512f_<code>v8qiv8di2<mask_name>_1"
17720 [(set (match_operand:V8DI 0 "register_operand" "=v")
17721 (any_extend:V8DI
17722 (match_operand:V8QI 1 "memory_operand" "m")))]
17723 "TARGET_AVX512F"
17724 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17725 [(set_attr "type" "ssemov")
17726 (set_attr "prefix" "evex")
17727 (set_attr "mode" "XI")])
17728
17729 (define_insn_and_split "*avx512f_<code>v8qiv8di2<mask_name>_2"
17730 [(set (match_operand:V8DI 0 "register_operand")
17731 (any_extend:V8DI
17732 (vec_select:V8QI
17733 (subreg:V16QI
17734 (vec_concat:V2DI
17735 (match_operand:DI 1 "memory_operand")
17736 (const_int 0)) 0)
17737 (parallel [(const_int 0) (const_int 1)
17738 (const_int 2) (const_int 3)
17739 (const_int 4) (const_int 5)
17740 (const_int 6) (const_int 7)]))))]
17741 "TARGET_AVX512F && ix86_pre_reload_split ()"
17742 "#"
17743 "&& 1"
17744 [(set (match_dup 0)
17745 (any_extend:V8DI (match_dup 1)))]
17746 "operands[1] = adjust_address_nv (operands[1], V8QImode, 0);")
15985 17747
15986 (define_insn "avx2_<code>v4qiv4di2<mask_name>" 17748 (define_insn "avx2_<code>v4qiv4di2<mask_name>"
15987 [(set (match_operand:V4DI 0 "register_operand" "=v") 17749 [(set (match_operand:V4DI 0 "register_operand" "=v")
15988 (any_extend:V4DI 17750 (any_extend:V4DI
15989 (vec_select:V4QI 17751 (vec_select:V4QI
15990 (match_operand:V16QI 1 "nonimmediate_operand" "vm") 17752 (match_operand:V16QI 1 "register_operand" "v")
15991 (parallel [(const_int 0) (const_int 1) 17753 (parallel [(const_int 0) (const_int 1)
15992 (const_int 2) (const_int 3)]))))] 17754 (const_int 2) (const_int 3)]))))]
15993 "TARGET_AVX2 && <mask_avx512vl_condition>" 17755 "TARGET_AVX2 && <mask_avx512vl_condition>"
15994 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}" 17756 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15995 [(set_attr "type" "ssemov") 17757 [(set_attr "type" "ssemov")
15996 (set_attr "prefix_extra" "1") 17758 (set_attr "prefix_extra" "1")
15997 (set_attr "prefix" "maybe_evex") 17759 (set_attr "prefix" "maybe_evex")
15998 (set_attr "mode" "OI")]) 17760 (set_attr "mode" "OI")])
15999 17761
17762 (define_insn "*avx2_<code>v4qiv4di2<mask_name>_1"
17763 [(set (match_operand:V4DI 0 "register_operand" "=v")
17764 (any_extend:V4DI
17765 (match_operand:V4QI 1 "memory_operand" "m")))]
17766 "TARGET_AVX2 && <mask_avx512vl_condition>"
17767 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17768 [(set_attr "type" "ssemov")
17769 (set_attr "prefix_extra" "1")
17770 (set_attr "prefix" "maybe_evex")
17771 (set_attr "mode" "OI")])
17772
17773 (define_insn_and_split "*avx2_<code>v4qiv4di2<mask_name>_2"
17774 [(set (match_operand:V4DI 0 "register_operand")
17775 (any_extend:V4DI
17776 (vec_select:V4QI
17777 (subreg:V16QI
17778 (vec_merge:V4SI
17779 (vec_duplicate:V4SI
17780 (match_operand:SI 1 "memory_operand"))
17781 (const_vector:V4SI
17782 [(const_int 0) (const_int 0)
17783 (const_int 0) (const_int 0)])
17784 (const_int 1)) 0)
17785 (parallel [(const_int 0) (const_int 1)
17786 (const_int 2) (const_int 3)]))))]
17787 "TARGET_AVX2 && <mask_avx512vl_condition>
17788 && ix86_pre_reload_split ()"
17789 "#"
17790 "&& 1"
17791 [(set (match_dup 0)
17792 (any_extend:V4DI (match_dup 1)))]
17793 "operands[1] = adjust_address_nv (operands[1], V4QImode, 0);")
17794
16000 (define_insn "sse4_1_<code>v2qiv2di2<mask_name>" 17795 (define_insn "sse4_1_<code>v2qiv2di2<mask_name>"
16001 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v") 17796 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
16002 (any_extend:V2DI 17797 (any_extend:V2DI
16003 (vec_select:V2QI 17798 (vec_select:V2QI
16004 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17799 (match_operand:V16QI 1 "register_operand" "Yr,*x,v")
16005 (parallel [(const_int 0) (const_int 1)]))))] 17800 (parallel [(const_int 0) (const_int 1)]))))]
16006 "TARGET_SSE4_1 && <mask_avx512vl_condition>" 17801 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
16007 "%vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %w1}" 17802 "%vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16008 [(set_attr "isa" "noavx,noavx,avx") 17803 [(set_attr "isa" "noavx,noavx,avx")
16009 (set_attr "type" "ssemov") 17804 (set_attr "type" "ssemov")
16010 (set_attr "prefix_extra" "1") 17805 (set_attr "prefix_extra" "1")
16011 (set_attr "prefix" "orig,orig,maybe_evex") 17806 (set_attr "prefix" "orig,orig,maybe_evex")
16012 (set_attr "mode" "TI")]) 17807 (set_attr "mode" "TI")])
16023 17818
16024 (define_insn "avx2_<code>v4hiv4di2<mask_name>" 17819 (define_insn "avx2_<code>v4hiv4di2<mask_name>"
16025 [(set (match_operand:V4DI 0 "register_operand" "=v") 17820 [(set (match_operand:V4DI 0 "register_operand" "=v")
16026 (any_extend:V4DI 17821 (any_extend:V4DI
16027 (vec_select:V4HI 17822 (vec_select:V4HI
16028 (match_operand:V8HI 1 "nonimmediate_operand" "vm") 17823 (match_operand:V8HI 1 "register_operand" "v")
16029 (parallel [(const_int 0) (const_int 1) 17824 (parallel [(const_int 0) (const_int 1)
16030 (const_int 2) (const_int 3)]))))] 17825 (const_int 2) (const_int 3)]))))]
16031 "TARGET_AVX2 && <mask_avx512vl_condition>" 17826 "TARGET_AVX2 && <mask_avx512vl_condition>"
16032 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}" 17827 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16033 [(set_attr "type" "ssemov") 17828 [(set_attr "type" "ssemov")
16034 (set_attr "prefix_extra" "1") 17829 (set_attr "prefix_extra" "1")
16035 (set_attr "prefix" "maybe_evex") 17830 (set_attr "prefix" "maybe_evex")
16036 (set_attr "mode" "OI")]) 17831 (set_attr "mode" "OI")])
16037 17832
17833 (define_insn "*avx2_<code>v4hiv4di2<mask_name>_1"
17834 [(set (match_operand:V4DI 0 "register_operand" "=v")
17835 (any_extend:V4DI
17836 (match_operand:V4HI 1 "memory_operand" "m")))]
17837 "TARGET_AVX2 && <mask_avx512vl_condition>"
17838 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17839 [(set_attr "type" "ssemov")
17840 (set_attr "prefix_extra" "1")
17841 (set_attr "prefix" "maybe_evex")
17842 (set_attr "mode" "OI")])
17843
17844 (define_insn_and_split "*avx2_<code>v4hiv4di2<mask_name>_2"
17845 [(set (match_operand:V4DI 0 "register_operand")
17846 (any_extend:V4DI
17847 (vec_select:V4HI
17848 (subreg:V8HI
17849 (vec_concat:V2DI
17850 (match_operand:DI 1 "memory_operand")
17851 (const_int 0)) 0)
17852 (parallel [(const_int 0) (const_int 1)
17853 (const_int 2) (const_int 3)]))))]
17854 "TARGET_AVX2 && <mask_avx512vl_condition>
17855 && ix86_pre_reload_split ()"
17856 "#"
17857 "&& 1"
17858 [(set (match_dup 0)
17859 (any_extend:V4DI (match_dup 1)))]
17860 "operands[1] = adjust_address_nv (operands[1], V4HImode, 0);")
17861
16038 (define_insn "sse4_1_<code>v2hiv2di2<mask_name>" 17862 (define_insn "sse4_1_<code>v2hiv2di2<mask_name>"
16039 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v") 17863 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
16040 (any_extend:V2DI 17864 (any_extend:V2DI
16041 (vec_select:V2HI 17865 (vec_select:V2HI
16042 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17866 (match_operand:V8HI 1 "register_operand" "Yr,*x,v")
16043 (parallel [(const_int 0) (const_int 1)]))))] 17867 (parallel [(const_int 0) (const_int 1)]))))]
16044 "TARGET_SSE4_1 && <mask_avx512vl_condition>" 17868 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
16045 "%vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}" 17869 "%vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16046 [(set_attr "isa" "noavx,noavx,avx") 17870 [(set_attr "isa" "noavx,noavx,avx")
16047 (set_attr "type" "ssemov") 17871 (set_attr "type" "ssemov")
16048 (set_attr "prefix_extra" "1") 17872 (set_attr "prefix_extra" "1")
16049 (set_attr "prefix" "orig,orig,maybe_evex") 17873 (set_attr "prefix" "orig,orig,maybe_evex")
16050 (set_attr "mode" "TI")]) 17874 (set_attr "mode" "TI")])
17875
17876 (define_insn "*sse4_1_<code>v2hiv2di2<mask_name>_1"
17877 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
17878 (any_extend:V2DI
17879 (match_operand:V2HI 1 "memory_operand" "m,m,m")))]
17880 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
17881 "%vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17882 [(set_attr "isa" "noavx,noavx,avx")
17883 (set_attr "type" "ssemov")
17884 (set_attr "prefix_extra" "1")
17885 (set_attr "prefix" "orig,orig,maybe_evex")
17886 (set_attr "mode" "TI")])
17887
17888 (define_insn_and_split "*sse4_1_<code>v2hiv2di2<mask_name>_2"
17889 [(set (match_operand:V2DI 0 "register_operand")
17890 (any_extend:V2DI
17891 (vec_select:V2HI
17892 (subreg:V8HI
17893 (vec_merge:V4SI
17894 (vec_duplicate:V4SI
17895 (match_operand:SI 1 "memory_operand"))
17896 (const_vector:V4SI
17897 [(const_int 0) (const_int 0)
17898 (const_int 0) (const_int 0)])
17899 (const_int 1)) 0)
17900 (parallel [(const_int 0) (const_int 1)]))))]
17901 "TARGET_SSE4_1 && <mask_avx512vl_condition>
17902 && ix86_pre_reload_split ()"
17903 "#"
17904 "&& 1"
17905 [(set (match_dup 0)
17906 (any_extend:V2DI (match_dup 1)))]
17907 "operands[1] = adjust_address_nv (operands[1], V2HImode, 0);")
16051 17908
16052 (define_insn "avx512f_<code>v8siv8di2<mask_name>" 17909 (define_insn "avx512f_<code>v8siv8di2<mask_name>"
16053 [(set (match_operand:V8DI 0 "register_operand" "=v") 17910 [(set (match_operand:V8DI 0 "register_operand" "=v")
16054 (any_extend:V8DI 17911 (any_extend:V8DI
16055 (match_operand:V8SI 1 "nonimmediate_operand" "vm")))] 17912 (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
16072 17929
16073 (define_insn "sse4_1_<code>v2siv2di2<mask_name>" 17930 (define_insn "sse4_1_<code>v2siv2di2<mask_name>"
16074 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v") 17931 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
16075 (any_extend:V2DI 17932 (any_extend:V2DI
16076 (vec_select:V2SI 17933 (vec_select:V2SI
16077 (match_operand:V4SI 1 "nonimmediate_operand" "Yrm,*xm,vm") 17934 (match_operand:V4SI 1 "register_operand" "Yr,*x,v")
16078 (parallel [(const_int 0) (const_int 1)]))))] 17935 (parallel [(const_int 0) (const_int 1)]))))]
16079 "TARGET_SSE4_1 && <mask_avx512vl_condition>" 17936 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
16080 "%vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}" 17937 "%vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16081 [(set_attr "isa" "noavx,noavx,avx") 17938 [(set_attr "isa" "noavx,noavx,avx")
16082 (set_attr "type" "ssemov") 17939 (set_attr "type" "ssemov")
16083 (set_attr "prefix_extra" "1") 17940 (set_attr "prefix_extra" "1")
16084 (set_attr "prefix" "orig,orig,maybe_evex") 17941 (set_attr "prefix" "orig,orig,maybe_evex")
16085 (set_attr "mode" "TI")]) 17942 (set_attr "mode" "TI")])
17943
17944 (define_insn "*sse4_1_<code>v2siv2di2<mask_name>_1"
17945 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
17946 (any_extend:V2DI
17947 (match_operand:V2SI 1 "memory_operand" "m,m,m")))]
17948 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
17949 "%vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17950 [(set_attr "isa" "noavx,noavx,avx")
17951 (set_attr "type" "ssemov")
17952 (set_attr "prefix_extra" "1")
17953 (set_attr "prefix" "orig,orig,maybe_evex")
17954 (set_attr "mode" "TI")])
17955
17956 (define_insn_and_split "*sse4_1_<code>v2siv2di2<mask_name>_2"
17957 [(set (match_operand:V2DI 0 "register_operand")
17958 (any_extend:V2DI
17959 (vec_select:V2SI
17960 (subreg:V4SI
17961 (vec_concat:V2DI
17962 (match_operand:DI 1 "memory_operand")
17963 (const_int 0)) 0)
17964 (parallel [(const_int 0) (const_int 1)]))))]
17965 "TARGET_SSE4_1 && <mask_avx512vl_condition>
17966 && ix86_pre_reload_split ()"
17967 "#"
17968 "&& 1"
17969 [(set (match_dup 0)
17970 (any_extend:V2DI (match_dup 1)))]
17971 "operands[1] = adjust_address_nv (operands[1], V2SImode, 0);")
16086 17972
16087 ;; ptestps/ptestpd are very similar to comiss and ucomiss when 17973 ;; ptestps/ptestpd are very similar to comiss and ucomiss when
16088 ;; setting FLAGS_REG. But it is not a really compare instruction. 17974 ;; setting FLAGS_REG. But it is not a really compare instruction.
16089 (define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>" 17975 (define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>"
16090 [(set (reg:CC FLAGS_REG) 17976 [(set (reg:CC FLAGS_REG)
16129 (set_attr "type" "ssecomi") 18015 (set_attr "type" "ssecomi")
16130 (set_attr "prefix_extra" "1") 18016 (set_attr "prefix_extra" "1")
16131 (set_attr "prefix" "orig,orig,vex") 18017 (set_attr "prefix" "orig,orig,vex")
16132 (set_attr "mode" "TI")]) 18018 (set_attr "mode" "TI")])
16133 18019
18020 (define_expand "nearbyint<mode>2"
18021 [(set (match_operand:VF 0 "register_operand")
18022 (unspec:VF
18023 [(match_operand:VF 1 "vector_operand")
18024 (match_dup 2)]
18025 UNSPEC_ROUND))]
18026 "TARGET_SSE4_1"
18027 "operands[2] = GEN_INT (ROUND_MXCSR | ROUND_NO_EXC);")
18028
18029 (define_expand "rint<mode>2"
18030 [(set (match_operand:VF 0 "register_operand")
18031 (unspec:VF
18032 [(match_operand:VF 1 "vector_operand")
18033 (match_dup 2)]
18034 UNSPEC_ROUND))]
18035 "TARGET_SSE4_1"
18036 "operands[2] = GEN_INT (ROUND_MXCSR);")
18037
16134 (define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>" 18038 (define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>"
16135 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x") 18039 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
16136 (unspec:VF_128_256 18040 (unspec:VF_128_256
16137 [(match_operand:VF_128_256 1 "vector_operand" "YrBm,*xBm,xm") 18041 [(match_operand:VF_128_256 1 "vector_operand" "YrBm,*xBm,xm")
16138 (match_operand:SI 2 "const_0_to_15_operand" "n,n,n")] 18042 (match_operand:SI 2 "const_0_to_15_operand" "n,n,n")]
16225 18129
16226 (define_insn "sse4_1_round<ssescalarmodesuffix>" 18130 (define_insn "sse4_1_round<ssescalarmodesuffix>"
16227 [(set (match_operand:VF_128 0 "register_operand" "=Yr,*x,x,v") 18131 [(set (match_operand:VF_128 0 "register_operand" "=Yr,*x,x,v")
16228 (vec_merge:VF_128 18132 (vec_merge:VF_128
16229 (unspec:VF_128 18133 (unspec:VF_128
16230 [(match_operand:VF_128 2 "register_operand" "Yr,*x,x,v") 18134 [(match_operand:VF_128 2 "nonimmediate_operand" "Yrm,*xm,xm,vm")
16231 (match_operand:SI 3 "const_0_to_15_operand" "n,n,n,n")] 18135 (match_operand:SI 3 "const_0_to_15_operand" "n,n,n,n")]
16232 UNSPEC_ROUND) 18136 UNSPEC_ROUND)
18137 (match_operand:VF_128 1 "register_operand" "0,0,x,v")
18138 (const_int 1)))]
18139 "TARGET_SSE4_1"
18140 "@
18141 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %3}
18142 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %3}
18143 vround<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %3}
18144 vrndscale<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %3}"
18145 [(set_attr "isa" "noavx,noavx,avx,avx512f")
18146 (set_attr "type" "ssecvt")
18147 (set_attr "length_immediate" "1")
18148 (set_attr "prefix_data16" "1,1,*,*")
18149 (set_attr "prefix_extra" "1")
18150 (set_attr "prefix" "orig,orig,vex,evex")
18151 (set_attr "mode" "<MODE>")])
18152
18153 (define_insn "*sse4_1_round<ssescalarmodesuffix>"
18154 [(set (match_operand:VF_128 0 "register_operand" "=Yr,*x,x,v")
18155 (vec_merge:VF_128
18156 (vec_duplicate:VF_128
18157 (unspec:<ssescalarmode>
18158 [(match_operand:<ssescalarmode> 2 "nonimmediate_operand" "Yrm,*xm,xm,vm")
18159 (match_operand:SI 3 "const_0_to_15_operand" "n,n,n,n")]
18160 UNSPEC_ROUND))
16233 (match_operand:VF_128 1 "register_operand" "0,0,x,v") 18161 (match_operand:VF_128 1 "register_operand" "0,0,x,v")
16234 (const_int 1)))] 18162 (const_int 1)))]
16235 "TARGET_SSE4_1" 18163 "TARGET_SSE4_1"
16236 "@ 18164 "@
16237 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3} 18165 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
16358 (match_dup 4) 18286 (match_dup 4)
16359 (match_dup 5) 18287 (match_dup 5)
16360 (match_dup 6)] 18288 (match_dup 6)]
16361 UNSPEC_PCMPESTR))] 18289 UNSPEC_PCMPESTR))]
16362 "TARGET_SSE4_2 18290 "TARGET_SSE4_2
16363 && can_create_pseudo_p ()" 18291 && ix86_pre_reload_split ()"
16364 "#" 18292 "#"
16365 "&& 1" 18293 "&& 1"
16366 [(const_int 0)] 18294 [(const_int 0)]
16367 { 18295 {
16368 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0])); 18296 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
16494 [(match_dup 2) 18422 [(match_dup 2)
16495 (match_dup 3) 18423 (match_dup 3)
16496 (match_dup 4)] 18424 (match_dup 4)]
16497 UNSPEC_PCMPISTR))] 18425 UNSPEC_PCMPISTR))]
16498 "TARGET_SSE4_2 18426 "TARGET_SSE4_2
16499 && can_create_pseudo_p ()" 18427 && ix86_pre_reload_split ()"
16500 "#" 18428 "#"
16501 "&& 1" 18429 "&& 1"
16502 [(const_int 0)] 18430 [(const_int 0)]
16503 { 18431 {
16504 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0])); 18432 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
16618 operands[5] 18546 operands[5]
16619 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1], 18547 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16620 operands[3]), UNSPEC_VSIBADDR); 18548 operands[3]), UNSPEC_VSIBADDR);
16621 }) 18549 })
16622 18550
16623 (define_insn "*avx512pf_gatherpf<mode>sf_mask" 18551 (define_insn "*avx512pf_gatherpf<VI48_512:mode>sf_mask"
16624 [(unspec 18552 [(unspec
16625 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk") 18553 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16626 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator" 18554 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
16627 [(unspec:P 18555 [(unspec:P
16628 [(match_operand:P 2 "vsib_address_operand" "Tv") 18556 [(match_operand:P 2 "vsib_address_operand" "Tv")
16634 "TARGET_AVX512PF" 18562 "TARGET_AVX512PF"
16635 { 18563 {
16636 switch (INTVAL (operands[4])) 18564 switch (INTVAL (operands[4]))
16637 { 18565 {
16638 case 3: 18566 case 3:
16639 return "vgatherpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}"; 18567 /* %X5 so that we don't emit any *WORD PTR for -masm=intel, as
18568 gas changed what it requires incompatibly. */
18569 return "%M2vgatherpf0<ssemodesuffix>ps\t{%5%{%0%}|%X5%{%0%}}";
16640 case 2: 18570 case 2:
16641 return "vgatherpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}"; 18571 return "%M2vgatherpf1<ssemodesuffix>ps\t{%5%{%0%}|%X5%{%0%}}";
16642 default: 18572 default:
16643 gcc_unreachable (); 18573 gcc_unreachable ();
16644 } 18574 }
16645 } 18575 }
16646 [(set_attr "type" "sse") 18576 [(set_attr "type" "sse")
16663 operands[5] 18593 operands[5]
16664 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1], 18594 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16665 operands[3]), UNSPEC_VSIBADDR); 18595 operands[3]), UNSPEC_VSIBADDR);
16666 }) 18596 })
16667 18597
16668 (define_insn "*avx512pf_gatherpf<mode>df_mask" 18598 (define_insn "*avx512pf_gatherpf<VI4_256_8_512:mode>df_mask"
16669 [(unspec 18599 [(unspec
16670 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk") 18600 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16671 (match_operator:V8DF 5 "vsib_mem_operator" 18601 (match_operator:V8DF 5 "vsib_mem_operator"
16672 [(unspec:P 18602 [(unspec:P
16673 [(match_operand:P 2 "vsib_address_operand" "Tv") 18603 [(match_operand:P 2 "vsib_address_operand" "Tv")
16679 "TARGET_AVX512PF" 18609 "TARGET_AVX512PF"
16680 { 18610 {
16681 switch (INTVAL (operands[4])) 18611 switch (INTVAL (operands[4]))
16682 { 18612 {
16683 case 3: 18613 case 3:
16684 return "vgatherpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}"; 18614 /* %X5 so that we don't emit any *WORD PTR for -masm=intel, as
18615 gas changed what it requires incompatibly. */
18616 return "%M2vgatherpf0<ssemodesuffix>pd\t{%5%{%0%}|%X5%{%0%}}";
16685 case 2: 18617 case 2:
16686 return "vgatherpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}"; 18618 return "%M2vgatherpf1<ssemodesuffix>pd\t{%5%{%0%}|%X5%{%0%}}";
16687 default: 18619 default:
16688 gcc_unreachable (); 18620 gcc_unreachable ();
16689 } 18621 }
16690 } 18622 }
16691 [(set_attr "type" "sse") 18623 [(set_attr "type" "sse")
16708 operands[5] 18640 operands[5]
16709 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1], 18641 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16710 operands[3]), UNSPEC_VSIBADDR); 18642 operands[3]), UNSPEC_VSIBADDR);
16711 }) 18643 })
16712 18644
16713 (define_insn "*avx512pf_scatterpf<mode>sf_mask" 18645 (define_insn "*avx512pf_scatterpf<VI48_512:mode>sf_mask"
16714 [(unspec 18646 [(unspec
16715 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk") 18647 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16716 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator" 18648 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
16717 [(unspec:P 18649 [(unspec:P
16718 [(match_operand:P 2 "vsib_address_operand" "Tv") 18650 [(match_operand:P 2 "vsib_address_operand" "Tv")
16725 { 18657 {
16726 switch (INTVAL (operands[4])) 18658 switch (INTVAL (operands[4]))
16727 { 18659 {
16728 case 3: 18660 case 3:
16729 case 7: 18661 case 7:
16730 return "vscatterpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}"; 18662 /* %X5 so that we don't emit any *WORD PTR for -masm=intel, as
18663 gas changed what it requires incompatibly. */
18664 return "%M2vscatterpf0<ssemodesuffix>ps\t{%5%{%0%}|%X5%{%0%}}";
16731 case 2: 18665 case 2:
16732 case 6: 18666 case 6:
16733 return "vscatterpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}"; 18667 return "%M2vscatterpf1<ssemodesuffix>ps\t{%5%{%0%}|%X5%{%0%}}";
16734 default: 18668 default:
16735 gcc_unreachable (); 18669 gcc_unreachable ();
16736 } 18670 }
16737 } 18671 }
16738 [(set_attr "type" "sse") 18672 [(set_attr "type" "sse")
16755 operands[5] 18689 operands[5]
16756 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1], 18690 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16757 operands[3]), UNSPEC_VSIBADDR); 18691 operands[3]), UNSPEC_VSIBADDR);
16758 }) 18692 })
16759 18693
16760 (define_insn "*avx512pf_scatterpf<mode>df_mask" 18694 (define_insn "*avx512pf_scatterpf<VI4_256_8_512:mode>df_mask"
16761 [(unspec 18695 [(unspec
16762 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk") 18696 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16763 (match_operator:V8DF 5 "vsib_mem_operator" 18697 (match_operator:V8DF 5 "vsib_mem_operator"
16764 [(unspec:P 18698 [(unspec:P
16765 [(match_operand:P 2 "vsib_address_operand" "Tv") 18699 [(match_operand:P 2 "vsib_address_operand" "Tv")
16772 { 18706 {
16773 switch (INTVAL (operands[4])) 18707 switch (INTVAL (operands[4]))
16774 { 18708 {
16775 case 3: 18709 case 3:
16776 case 7: 18710 case 7:
16777 return "vscatterpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}"; 18711 /* %X5 so that we don't emit any *WORD PTR for -masm=intel, as
18712 gas changed what it requires incompatibly. */
18713 return "%M2vscatterpf0<ssemodesuffix>pd\t{%5%{%0%}|%X5%{%0%}}";
16778 case 2: 18714 case 2:
16779 case 6: 18715 case 6:
16780 return "vscatterpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}"; 18716 return "%M2vscatterpf1<ssemodesuffix>pd\t{%5%{%0%}|%X5%{%0%}}";
16781 default: 18717 default:
16782 gcc_unreachable (); 18718 gcc_unreachable ();
16783 } 18719 }
16784 } 18720 }
16785 [(set_attr "type" "sse") 18721 [(set_attr "type" "sse")
16810 18746
16811 (define_insn "avx512er_vmrcp28<mode><round_saeonly_name>" 18747 (define_insn "avx512er_vmrcp28<mode><round_saeonly_name>"
16812 [(set (match_operand:VF_128 0 "register_operand" "=v") 18748 [(set (match_operand:VF_128 0 "register_operand" "=v")
16813 (vec_merge:VF_128 18749 (vec_merge:VF_128
16814 (unspec:VF_128 18750 (unspec:VF_128
16815 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")] 18751 [(match_operand:VF_128 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")]
16816 UNSPEC_RCP28) 18752 UNSPEC_RCP28)
16817 (match_operand:VF_128 2 "register_operand" "v") 18753 (match_operand:VF_128 2 "register_operand" "v")
16818 (const_int 1)))] 18754 (const_int 1)))]
16819 "TARGET_AVX512ER" 18755 "TARGET_AVX512ER"
16820 "vrcp28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %<iptr>1<round_saeonly_op3>}" 18756 "vrcp28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %<iptr>1<round_saeonly_op3>}"
16836 18772
16837 (define_insn "avx512er_vmrsqrt28<mode><round_saeonly_name>" 18773 (define_insn "avx512er_vmrsqrt28<mode><round_saeonly_name>"
16838 [(set (match_operand:VF_128 0 "register_operand" "=v") 18774 [(set (match_operand:VF_128 0 "register_operand" "=v")
16839 (vec_merge:VF_128 18775 (vec_merge:VF_128
16840 (unspec:VF_128 18776 (unspec:VF_128
16841 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")] 18777 [(match_operand:VF_128 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")]
16842 UNSPEC_RSQRT28) 18778 UNSPEC_RSQRT28)
16843 (match_operand:VF_128 2 "register_operand" "v") 18779 (match_operand:VF_128 2 "register_operand" "v")
16844 (const_int 1)))] 18780 (const_int 1)))]
16845 "TARGET_AVX512ER" 18781 "TARGET_AVX512ER"
16846 "vrsqrt28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %<iptr>1<round_saeonly_op3>}" 18782 "vrsqrt28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %<iptr>1<round_saeonly_op3>}"
17884 (set_attr "prefix" "vex") 19820 (set_attr "prefix" "vex")
17885 (set_attr "btver2_decode" "vector") 19821 (set_attr "btver2_decode" "vector")
17886 (set_attr "mode" "OI")]) 19822 (set_attr "mode" "OI")])
17887 19823
17888 ;; Clear the upper 128bits of AVX registers, equivalent to a NOP 19824 ;; Clear the upper 128bits of AVX registers, equivalent to a NOP
17889 ;; if the upper 128bits are unused. 19825 ;; if the upper 128bits are unused. Initially we expand the instructions
17890 (define_insn "avx_vzeroupper" 19826 ;; as though they had no effect on the SSE registers, but later add SETs and
17891 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)] 19827 ;; CLOBBERs to the PARALLEL to model the real effect.
17892 "TARGET_AVX" 19828 (define_expand "avx_vzeroupper"
19829 [(parallel [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)])]
19830 "TARGET_AVX")
19831
19832 (define_insn "*avx_vzeroupper"
19833 [(match_parallel 0 "vzeroupper_pattern"
19834 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)])]
19835 "TARGET_AVX && XVECLEN (operands[0], 0) == (TARGET_64BIT ? 16 : 8) + 1"
17893 "vzeroupper" 19836 "vzeroupper"
19837 [(set_attr "type" "sse")
19838 (set_attr "modrm" "0")
19839 (set_attr "memory" "none")
19840 (set_attr "prefix" "vex")
19841 (set_attr "btver2_decode" "vector")
19842 (set_attr "mode" "OI")])
19843
19844 (define_insn_and_split "*avx_vzeroupper_1"
19845 [(match_parallel 0 "vzeroupper_pattern"
19846 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)])]
19847 "TARGET_AVX && XVECLEN (operands[0], 0) != (TARGET_64BIT ? 16 : 8) + 1"
19848 "#"
19849 "&& epilogue_completed"
19850 [(match_dup 0)]
19851 {
19852 /* For IPA-RA purposes, make it clear the instruction clobbers
19853 even XMM registers not mentioned explicitly in the pattern. */
19854 unsigned int nregs = TARGET_64BIT ? 16 : 8;
19855 unsigned int npats = XVECLEN (operands[0], 0);
19856 rtvec vec = rtvec_alloc (nregs + 1);
19857 RTVEC_ELT (vec, 0) = XVECEXP (operands[0], 0, 0);
19858 for (unsigned int i = 0, j = 1; i < nregs; ++i)
19859 {
19860 unsigned int regno = GET_SSE_REGNO (i);
19861 if (j < npats
19862 && REGNO (SET_DEST (XVECEXP (operands[0], 0, j))) == regno)
19863 {
19864 RTVEC_ELT (vec, i + 1) = XVECEXP (operands[0], 0, j);
19865 j++;
19866 }
19867 else
19868 {
19869 rtx reg = gen_rtx_REG (V2DImode, regno);
19870 RTVEC_ELT (vec, i + 1) = gen_rtx_CLOBBER (VOIDmode, reg);
19871 }
19872 }
19873 operands[0] = gen_rtx_PARALLEL (VOIDmode, vec);
19874 }
17894 [(set_attr "type" "sse") 19875 [(set_attr "type" "sse")
17895 (set_attr "modrm" "0") 19876 (set_attr "modrm" "0")
17896 (set_attr "memory" "none") 19877 (set_attr "memory" "none")
17897 (set_attr "prefix" "vex") 19878 (set_attr "prefix" "vex")
17898 (set_attr "btver2_decode" "vector") 19879 (set_attr "btver2_decode" "vector")
17968 UNSPEC_VPERMVAR))] 19949 UNSPEC_VPERMVAR))]
17969 "TARGET_AVX512BW && <mask_mode512bit_condition>" 19950 "TARGET_AVX512BW && <mask_mode512bit_condition>"
17970 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}" 19951 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
17971 [(set_attr "type" "sselog") 19952 [(set_attr "type" "sselog")
17972 (set_attr "prefix" "<mask_prefix2>") 19953 (set_attr "prefix" "<mask_prefix2>")
19954 (set_attr "mode" "<sseinsnmode>")])
19955
19956 ;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
19957 ;; If it so happens that the input is in memory, use vbroadcast.
19958 ;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
19959 (define_insn "*avx_vperm_broadcast_v4sf"
19960 [(set (match_operand:V4SF 0 "register_operand" "=v,v,v")
19961 (vec_select:V4SF
19962 (match_operand:V4SF 1 "nonimmediate_operand" "m,o,v")
19963 (match_parallel 2 "avx_vbroadcast_operand"
19964 [(match_operand 3 "const_int_operand" "C,n,n")])))]
19965 "TARGET_AVX"
19966 {
19967 int elt = INTVAL (operands[3]);
19968 switch (which_alternative)
19969 {
19970 case 0:
19971 case 1:
19972 operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
19973 return "vbroadcastss\t{%1, %0|%0, %k1}";
19974 case 2:
19975 operands[2] = GEN_INT (elt * 0x55);
19976 return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
19977 default:
19978 gcc_unreachable ();
19979 }
19980 }
19981 [(set_attr "type" "ssemov,ssemov,sselog1")
19982 (set_attr "prefix_extra" "1")
19983 (set_attr "length_immediate" "0,0,1")
19984 (set_attr "prefix" "maybe_evex")
19985 (set_attr "mode" "SF,SF,V4SF")])
19986
19987 (define_insn_and_split "*avx_vperm_broadcast_<mode>"
19988 [(set (match_operand:VF_256 0 "register_operand" "=v,v,v")
19989 (vec_select:VF_256
19990 (match_operand:VF_256 1 "nonimmediate_operand" "m,o,?v")
19991 (match_parallel 2 "avx_vbroadcast_operand"
19992 [(match_operand 3 "const_int_operand" "C,n,n")])))]
19993 "TARGET_AVX
19994 && (<MODE>mode != V4DFmode || !TARGET_AVX2 || operands[3] == const0_rtx)"
19995 "#"
19996 "&& reload_completed"
19997 [(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
19998 {
19999 rtx op0 = operands[0], op1 = operands[1];
20000 int elt = INTVAL (operands[3]);
20001
20002 if (REG_P (op1))
20003 {
20004 int mask;
20005
20006 if (TARGET_AVX2 && elt == 0)
20007 {
20008 emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
20009 op1)));
20010 DONE;
20011 }
20012
20013 /* Shuffle element we care about into all elements of the 128-bit lane.
20014 The other lane gets shuffled too, but we don't care. */
20015 if (<MODE>mode == V4DFmode)
20016 mask = (elt & 1 ? 15 : 0);
20017 else
20018 mask = (elt & 3) * 0x55;
20019 emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
20020
20021 /* Shuffle the lane we care about into both lanes of the dest. */
20022 mask = (elt / (<ssescalarnum> / 2)) * 0x11;
20023 if (EXT_REX_SSE_REG_P (op0))
20024 {
20025 /* There is no EVEX VPERM2F128, but we can use either VBROADCASTSS
20026 or VSHUFF128. */
20027 gcc_assert (<MODE>mode == V8SFmode);
20028 if ((mask & 1) == 0)
20029 emit_insn (gen_avx2_vec_dupv8sf (op0,
20030 gen_lowpart (V4SFmode, op0)));
20031 else
20032 emit_insn (gen_avx512vl_shuf_f32x4_1 (op0, op0, op0,
20033 GEN_INT (4), GEN_INT (5),
20034 GEN_INT (6), GEN_INT (7),
20035 GEN_INT (12), GEN_INT (13),
20036 GEN_INT (14), GEN_INT (15)));
20037 DONE;
20038 }
20039
20040 emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
20041 DONE;
20042 }
20043
20044 operands[1] = adjust_address (op1, <ssescalarmode>mode,
20045 elt * GET_MODE_SIZE (<ssescalarmode>mode));
20046 })
20047
20048 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
20049 [(set (match_operand:VF2 0 "register_operand")
20050 (vec_select:VF2
20051 (match_operand:VF2 1 "nonimmediate_operand")
20052 (match_operand:SI 2 "const_0_to_255_operand")))]
20053 "TARGET_AVX && <mask_mode512bit_condition>"
20054 {
20055 int mask = INTVAL (operands[2]);
20056 rtx perm[<ssescalarnum>];
20057
20058 int i;
20059 for (i = 0; i < <ssescalarnum>; i = i + 2)
20060 {
20061 perm[i] = GEN_INT (((mask >> i) & 1) + i);
20062 perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
20063 }
20064
20065 operands[2]
20066 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
20067 })
20068
20069 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
20070 [(set (match_operand:VF1 0 "register_operand")
20071 (vec_select:VF1
20072 (match_operand:VF1 1 "nonimmediate_operand")
20073 (match_operand:SI 2 "const_0_to_255_operand")))]
20074 "TARGET_AVX && <mask_mode512bit_condition>"
20075 {
20076 int mask = INTVAL (operands[2]);
20077 rtx perm[<ssescalarnum>];
20078
20079 int i;
20080 for (i = 0; i < <ssescalarnum>; i = i + 4)
20081 {
20082 perm[i] = GEN_INT (((mask >> 0) & 3) + i);
20083 perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
20084 perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
20085 perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
20086 }
20087
20088 operands[2]
20089 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
20090 })
20091
20092 ;; This pattern needs to come before the avx2_perm*/avx512f_perm*
20093 ;; patterns, as they have the same RTL representation (vpermilp*
20094 ;; being a subset of what vpermp* can do), but vpermilp* has shorter
20095 ;; latency as it never crosses lanes.
20096 (define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
20097 [(set (match_operand:VF 0 "register_operand" "=v")
20098 (vec_select:VF
20099 (match_operand:VF 1 "nonimmediate_operand" "vm")
20100 (match_parallel 2 ""
20101 [(match_operand 3 "const_int_operand")])))]
20102 "TARGET_AVX && <mask_mode512bit_condition>
20103 && avx_vpermilp_parallel (operands[2], <MODE>mode)"
20104 {
20105 int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
20106 operands[2] = GEN_INT (mask);
20107 return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
20108 }
20109 [(set_attr "type" "sselog")
20110 (set_attr "prefix_extra" "1")
20111 (set_attr "length_immediate" "1")
20112 (set_attr "prefix" "<mask_prefix>")
17973 (set_attr "mode" "<sseinsnmode>")]) 20113 (set_attr "mode" "<sseinsnmode>")])
17974 20114
17975 (define_expand "avx2_perm<mode>" 20115 (define_expand "avx2_perm<mode>"
17976 [(match_operand:VI8F_256 0 "register_operand") 20116 [(match_operand:VI8F_256 0 "register_operand")
17977 (match_operand:VI8F_256 1 "nonimmediate_operand") 20117 (match_operand:VI8F_256 1 "nonimmediate_operand")
18453 20593
18454 (define_insn "avx512cd_maskb_vec_dup<mode>" 20594 (define_insn "avx512cd_maskb_vec_dup<mode>"
18455 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v") 20595 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
18456 (vec_duplicate:VI8_AVX512VL 20596 (vec_duplicate:VI8_AVX512VL
18457 (zero_extend:DI 20597 (zero_extend:DI
18458 (match_operand:QI 1 "register_operand" "Yk"))))] 20598 (match_operand:QI 1 "register_operand" "k"))))]
18459 "TARGET_AVX512CD" 20599 "TARGET_AVX512CD"
18460 "vpbroadcastmb2q\t{%1, %0|%0, %1}" 20600 "vpbroadcastmb2q\t{%1, %0|%0, %1}"
18461 [(set_attr "type" "mskmov") 20601 [(set_attr "type" "mskmov")
18462 (set_attr "prefix" "evex") 20602 (set_attr "prefix" "evex")
18463 (set_attr "mode" "XI")]) 20603 (set_attr "mode" "XI")])
18464 20604
18465 (define_insn "avx512cd_maskw_vec_dup<mode>" 20605 (define_insn "avx512cd_maskw_vec_dup<mode>"
18466 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v") 20606 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
18467 (vec_duplicate:VI4_AVX512VL 20607 (vec_duplicate:VI4_AVX512VL
18468 (zero_extend:SI 20608 (zero_extend:SI
18469 (match_operand:HI 1 "register_operand" "Yk"))))] 20609 (match_operand:HI 1 "register_operand" "k"))))]
18470 "TARGET_AVX512CD" 20610 "TARGET_AVX512CD"
18471 "vpbroadcastmw2d\t{%1, %0|%0, %1}" 20611 "vpbroadcastmw2d\t{%1, %0|%0, %1}"
18472 [(set_attr "type" "mskmov") 20612 [(set_attr "type" "mskmov")
18473 (set_attr "prefix" "evex") 20613 (set_attr "prefix" "evex")
18474 (set_attr "mode" "XI")]) 20614 (set_attr "mode" "XI")])
18475
18476 ;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
18477 ;; If it so happens that the input is in memory, use vbroadcast.
18478 ;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
18479 (define_insn "*avx_vperm_broadcast_v4sf"
18480 [(set (match_operand:V4SF 0 "register_operand" "=v,v,v")
18481 (vec_select:V4SF
18482 (match_operand:V4SF 1 "nonimmediate_operand" "m,o,v")
18483 (match_parallel 2 "avx_vbroadcast_operand"
18484 [(match_operand 3 "const_int_operand" "C,n,n")])))]
18485 "TARGET_AVX"
18486 {
18487 int elt = INTVAL (operands[3]);
18488 switch (which_alternative)
18489 {
18490 case 0:
18491 case 1:
18492 operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
18493 return "vbroadcastss\t{%1, %0|%0, %k1}";
18494 case 2:
18495 operands[2] = GEN_INT (elt * 0x55);
18496 return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
18497 default:
18498 gcc_unreachable ();
18499 }
18500 }
18501 [(set_attr "type" "ssemov,ssemov,sselog1")
18502 (set_attr "prefix_extra" "1")
18503 (set_attr "length_immediate" "0,0,1")
18504 (set_attr "prefix" "maybe_evex")
18505 (set_attr "mode" "SF,SF,V4SF")])
18506
18507 (define_insn_and_split "*avx_vperm_broadcast_<mode>"
18508 [(set (match_operand:VF_256 0 "register_operand" "=v,v,v")
18509 (vec_select:VF_256
18510 (match_operand:VF_256 1 "nonimmediate_operand" "m,o,?v")
18511 (match_parallel 2 "avx_vbroadcast_operand"
18512 [(match_operand 3 "const_int_operand" "C,n,n")])))]
18513 "TARGET_AVX"
18514 "#"
18515 "&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
18516 [(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
18517 {
18518 rtx op0 = operands[0], op1 = operands[1];
18519 int elt = INTVAL (operands[3]);
18520
18521 if (REG_P (op1))
18522 {
18523 int mask;
18524
18525 if (TARGET_AVX2 && elt == 0)
18526 {
18527 emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
18528 op1)));
18529 DONE;
18530 }
18531
18532 /* Shuffle element we care about into all elements of the 128-bit lane.
18533 The other lane gets shuffled too, but we don't care. */
18534 if (<MODE>mode == V4DFmode)
18535 mask = (elt & 1 ? 15 : 0);
18536 else
18537 mask = (elt & 3) * 0x55;
18538 emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
18539
18540 /* Shuffle the lane we care about into both lanes of the dest. */
18541 mask = (elt / (<ssescalarnum> / 2)) * 0x11;
18542 if (EXT_REX_SSE_REG_P (op0))
18543 {
18544 /* There is no EVEX VPERM2F128, but we can use either VBROADCASTSS
18545 or VSHUFF128. */
18546 gcc_assert (<MODE>mode == V8SFmode);
18547 if ((mask & 1) == 0)
18548 emit_insn (gen_avx2_vec_dupv8sf (op0,
18549 gen_lowpart (V4SFmode, op0)));
18550 else
18551 emit_insn (gen_avx512vl_shuf_f32x4_1 (op0, op0, op0,
18552 GEN_INT (4), GEN_INT (5),
18553 GEN_INT (6), GEN_INT (7),
18554 GEN_INT (12), GEN_INT (13),
18555 GEN_INT (14), GEN_INT (15)));
18556 DONE;
18557 }
18558
18559 emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
18560 DONE;
18561 }
18562
18563 operands[1] = adjust_address (op1, <ssescalarmode>mode,
18564 elt * GET_MODE_SIZE (<ssescalarmode>mode));
18565 })
18566
18567 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
18568 [(set (match_operand:VF2 0 "register_operand")
18569 (vec_select:VF2
18570 (match_operand:VF2 1 "nonimmediate_operand")
18571 (match_operand:SI 2 "const_0_to_255_operand")))]
18572 "TARGET_AVX && <mask_mode512bit_condition>"
18573 {
18574 int mask = INTVAL (operands[2]);
18575 rtx perm[<ssescalarnum>];
18576
18577 int i;
18578 for (i = 0; i < <ssescalarnum>; i = i + 2)
18579 {
18580 perm[i] = GEN_INT (((mask >> i) & 1) + i);
18581 perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
18582 }
18583
18584 operands[2]
18585 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
18586 })
18587
18588 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
18589 [(set (match_operand:VF1 0 "register_operand")
18590 (vec_select:VF1
18591 (match_operand:VF1 1 "nonimmediate_operand")
18592 (match_operand:SI 2 "const_0_to_255_operand")))]
18593 "TARGET_AVX && <mask_mode512bit_condition>"
18594 {
18595 int mask = INTVAL (operands[2]);
18596 rtx perm[<ssescalarnum>];
18597
18598 int i;
18599 for (i = 0; i < <ssescalarnum>; i = i + 4)
18600 {
18601 perm[i] = GEN_INT (((mask >> 0) & 3) + i);
18602 perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
18603 perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
18604 perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
18605 }
18606
18607 operands[2]
18608 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
18609 })
18610
18611 (define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
18612 [(set (match_operand:VF 0 "register_operand" "=v")
18613 (vec_select:VF
18614 (match_operand:VF 1 "nonimmediate_operand" "vm")
18615 (match_parallel 2 ""
18616 [(match_operand 3 "const_int_operand")])))]
18617 "TARGET_AVX && <mask_mode512bit_condition>
18618 && avx_vpermilp_parallel (operands[2], <MODE>mode)"
18619 {
18620 int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
18621 operands[2] = GEN_INT (mask);
18622 return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
18623 }
18624 [(set_attr "type" "sselog")
18625 (set_attr "prefix_extra" "1")
18626 (set_attr "length_immediate" "1")
18627 (set_attr "prefix" "<mask_prefix>")
18628 (set_attr "mode" "<sseinsnmode>")])
18629 20615
18630 (define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3<mask_name>" 20616 (define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3<mask_name>"
18631 [(set (match_operand:VF 0 "register_operand" "=v") 20617 [(set (match_operand:VF 0 "register_operand" "=v")
18632 (unspec:VF 20618 (unspec:VF
18633 [(match_operand:VF 1 "register_operand" "v") 20619 [(match_operand:VF 1 "register_operand" "v")
19183 }) 21169 })
19184 21170
19185 21171
19186 (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>" 21172 (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
19187 [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m") 21173 [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
19188 (unspec:AVX256MODE2P 21174 (vec_concat:AVX256MODE2P
19189 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")] 21175 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")
19190 UNSPEC_CAST))] 21176 (unspec:<ssehalfvecmode> [(const_int 0)] UNSPEC_CAST)))]
19191 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))" 21177 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
19192 "#" 21178 "#"
19193 "&& reload_completed" 21179 "&& reload_completed"
19194 [(set (match_dup 0) (match_dup 1))] 21180 [(set (match_dup 0) (match_dup 1))]
19195 { 21181 {
19284 (set_attr "mode" "<sseinsnmode>")]) 21270 (set_attr "mode" "<sseinsnmode>")])
19285 21271
19286 (define_insn "avx_vec_concat<mode>" 21272 (define_insn "avx_vec_concat<mode>"
19287 [(set (match_operand:V_256_512 0 "register_operand" "=x,v,x,Yv") 21273 [(set (match_operand:V_256_512 0 "register_operand" "=x,v,x,Yv")
19288 (vec_concat:V_256_512 21274 (vec_concat:V_256_512
19289 (match_operand:<ssehalfvecmode> 1 "register_operand" "x,v,x,v") 21275 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "x,v,xm,vm")
19290 (match_operand:<ssehalfvecmode> 2 "nonimm_or_0_operand" "xm,vm,C,C")))] 21276 (match_operand:<ssehalfvecmode> 2 "nonimm_or_0_operand" "xm,vm,C,C")))]
19291 "TARGET_AVX" 21277 "TARGET_AVX
21278 && (operands[2] == CONST0_RTX (<ssehalfvecmode>mode)
21279 || !MEM_P (operands[1]))"
19292 { 21280 {
19293 switch (which_alternative) 21281 switch (which_alternative)
19294 { 21282 {
19295 case 0: 21283 case 0:
19296 return "vinsert<i128>\t{$0x1, %2, %<xtg_mode>1, %0|%0, %<xtg_mode>1, %2, 0x1}"; 21284 return "vinsert<i128>\t{$0x1, %2, %<xtg_mode>1, %0|%0, %<xtg_mode>1, %2, 0x1}";
19312 case 2: 21300 case 2:
19313 case 3: 21301 case 3:
19314 switch (get_attr_mode (insn)) 21302 switch (get_attr_mode (insn))
19315 { 21303 {
19316 case MODE_V16SF: 21304 case MODE_V16SF:
19317 return "vmovaps\t{%1, %t0|%t0, %1}"; 21305 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
21306 return "vmovups\t{%1, %t0|%t0, %1}";
21307 else
21308 return "vmovaps\t{%1, %t0|%t0, %1}";
19318 case MODE_V8DF: 21309 case MODE_V8DF:
19319 return "vmovapd\t{%1, %t0|%t0, %1}"; 21310 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
21311 return "vmovupd\t{%1, %t0|%t0, %1}";
21312 else
21313 return "vmovapd\t{%1, %t0|%t0, %1}";
19320 case MODE_V8SF: 21314 case MODE_V8SF:
19321 return "vmovaps\t{%1, %x0|%x0, %1}"; 21315 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
21316 return "vmovups\t{%1, %x0|%x0, %1}";
21317 else
21318 return "vmovaps\t{%1, %x0|%x0, %1}";
19322 case MODE_V4DF: 21319 case MODE_V4DF:
19323 return "vmovapd\t{%1, %x0|%x0, %1}"; 21320 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
21321 return "vmovupd\t{%1, %x0|%x0, %1}";
21322 else
21323 return "vmovapd\t{%1, %x0|%x0, %1}";
19324 case MODE_XI: 21324 case MODE_XI:
19325 if (which_alternative == 2) 21325 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
19326 return "vmovdqa\t{%1, %t0|%t0, %1}"; 21326 {
19327 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) 21327 if (which_alternative == 2)
19328 return "vmovdqa64\t{%1, %t0|%t0, %1}"; 21328 return "vmovdqu\t{%1, %t0|%t0, %1}";
21329 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
21330 return "vmovdqu64\t{%1, %t0|%t0, %1}";
21331 else
21332 return "vmovdqu32\t{%1, %t0|%t0, %1}";
21333 }
19329 else 21334 else
19330 return "vmovdqa32\t{%1, %t0|%t0, %1}"; 21335 {
21336 if (which_alternative == 2)
21337 return "vmovdqa\t{%1, %t0|%t0, %1}";
21338 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
21339 return "vmovdqa64\t{%1, %t0|%t0, %1}";
21340 else
21341 return "vmovdqa32\t{%1, %t0|%t0, %1}";
21342 }
19331 case MODE_OI: 21343 case MODE_OI:
19332 if (which_alternative == 2) 21344 if (misaligned_operand (operands[1], <ssehalfvecmode>mode))
19333 return "vmovdqa\t{%1, %x0|%x0, %1}"; 21345 {
19334 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) 21346 if (which_alternative == 2)
19335 return "vmovdqa64\t{%1, %x0|%x0, %1}"; 21347 return "vmovdqu\t{%1, %x0|%x0, %1}";
21348 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
21349 return "vmovdqu64\t{%1, %x0|%x0, %1}";
21350 else
21351 return "vmovdqu32\t{%1, %x0|%x0, %1}";
21352 }
19336 else 21353 else
19337 return "vmovdqa32\t{%1, %x0|%x0, %1}"; 21354 {
21355 if (which_alternative == 2)
21356 return "vmovdqa\t{%1, %x0|%x0, %1}";
21357 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
21358 return "vmovdqa64\t{%1, %x0|%x0, %1}";
21359 else
21360 return "vmovdqa32\t{%1, %x0|%x0, %1}";
21361 }
19338 default: 21362 default:
19339 gcc_unreachable (); 21363 gcc_unreachable ();
19340 } 21364 }
19341 default: 21365 default:
19342 gcc_unreachable (); 21366 gcc_unreachable ();
19504 operands[6] 21528 operands[6]
19505 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], 21529 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19506 operands[5]), UNSPEC_VSIBADDR); 21530 operands[5]), UNSPEC_VSIBADDR);
19507 }) 21531 })
19508 21532
19509 (define_insn "*avx2_gathersi<mode>" 21533 (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>"
19510 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x") 21534 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19511 (unspec:VEC_GATHER_MODE 21535 (unspec:VEC_GATHER_MODE
19512 [(match_operand:VEC_GATHER_MODE 2 "register_operand" "0") 21536 [(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
19513 (match_operator:<ssescalarmode> 7 "vsib_mem_operator" 21537 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19514 [(unspec:P 21538 [(unspec:P
19519 (mem:BLK (scratch)) 21543 (mem:BLK (scratch))
19520 (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")] 21544 (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
19521 UNSPEC_GATHER)) 21545 UNSPEC_GATHER))
19522 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))] 21546 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19523 "TARGET_AVX2" 21547 "TARGET_AVX2"
19524 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}" 21548 "%M3v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
19525 [(set_attr "type" "ssemov") 21549 [(set_attr "type" "ssemov")
19526 (set_attr "prefix" "vex") 21550 (set_attr "prefix" "vex")
19527 (set_attr "mode" "<sseinsnmode>")]) 21551 (set_attr "mode" "<sseinsnmode>")])
19528 21552
19529 (define_insn "*avx2_gathersi<mode>_2" 21553 (define_insn "*avx2_gathersi<VEC_GATHER_MODE:mode>_2"
19530 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x") 21554 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19531 (unspec:VEC_GATHER_MODE 21555 (unspec:VEC_GATHER_MODE
19532 [(pc) 21556 [(pc)
19533 (match_operator:<ssescalarmode> 6 "vsib_mem_operator" 21557 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19534 [(unspec:P 21558 [(unspec:P
19539 (mem:BLK (scratch)) 21563 (mem:BLK (scratch))
19540 (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")] 21564 (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
19541 UNSPEC_GATHER)) 21565 UNSPEC_GATHER))
19542 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))] 21566 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19543 "TARGET_AVX2" 21567 "TARGET_AVX2"
19544 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}" 21568 "%M2v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}"
19545 [(set_attr "type" "ssemov") 21569 [(set_attr "type" "ssemov")
19546 (set_attr "prefix" "vex") 21570 (set_attr "prefix" "vex")
19547 (set_attr "mode" "<sseinsnmode>")]) 21571 (set_attr "mode" "<sseinsnmode>")])
19548 21572
19549 (define_expand "avx2_gatherdi<mode>" 21573 (define_expand "avx2_gatherdi<mode>"
19565 operands[6] 21589 operands[6]
19566 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], 21590 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19567 operands[5]), UNSPEC_VSIBADDR); 21591 operands[5]), UNSPEC_VSIBADDR);
19568 }) 21592 })
19569 21593
19570 (define_insn "*avx2_gatherdi<mode>" 21594 (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>"
19571 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x") 21595 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19572 (unspec:VEC_GATHER_MODE 21596 (unspec:VEC_GATHER_MODE
19573 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0") 21597 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
19574 (match_operator:<ssescalarmode> 7 "vsib_mem_operator" 21598 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19575 [(unspec:P 21599 [(unspec:P
19580 (mem:BLK (scratch)) 21604 (mem:BLK (scratch))
19581 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")] 21605 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
19582 UNSPEC_GATHER)) 21606 UNSPEC_GATHER))
19583 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))] 21607 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19584 "TARGET_AVX2" 21608 "TARGET_AVX2"
19585 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}" 21609 "%M3v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}"
19586 [(set_attr "type" "ssemov") 21610 [(set_attr "type" "ssemov")
19587 (set_attr "prefix" "vex") 21611 (set_attr "prefix" "vex")
19588 (set_attr "mode" "<sseinsnmode>")]) 21612 (set_attr "mode" "<sseinsnmode>")])
19589 21613
19590 (define_insn "*avx2_gatherdi<mode>_2" 21614 (define_insn "*avx2_gatherdi<VEC_GATHER_MODE:mode>_2"
19591 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x") 21615 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19592 (unspec:VEC_GATHER_MODE 21616 (unspec:VEC_GATHER_MODE
19593 [(pc) 21617 [(pc)
19594 (match_operator:<ssescalarmode> 6 "vsib_mem_operator" 21618 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19595 [(unspec:P 21619 [(unspec:P
19601 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")] 21625 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
19602 UNSPEC_GATHER)) 21626 UNSPEC_GATHER))
19603 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))] 21627 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19604 "TARGET_AVX2" 21628 "TARGET_AVX2"
19605 { 21629 {
19606 if (<MODE>mode != <VEC_GATHER_SRCDI>mode) 21630 if (<VEC_GATHER_MODE:MODE>mode != <VEC_GATHER_SRCDI>mode)
19607 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}"; 21631 return "%M2v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}";
19608 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"; 21632 return "%M2v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}";
19609 } 21633 }
19610 [(set_attr "type" "ssemov") 21634 [(set_attr "type" "ssemov")
19611 (set_attr "prefix" "vex") 21635 (set_attr "prefix" "vex")
19612 (set_attr "mode" "<sseinsnmode>")]) 21636 (set_attr "mode" "<sseinsnmode>")])
19613 21637
19614 (define_insn "*avx2_gatherdi<mode>_3" 21638 (define_insn "*avx2_gatherdi<VI4F_256:mode>_3"
19615 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x") 21639 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
19616 (vec_select:<VEC_GATHER_SRCDI> 21640 (vec_select:<VEC_GATHER_SRCDI>
19617 (unspec:VI4F_256 21641 (unspec:VI4F_256
19618 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0") 21642 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
19619 (match_operator:<ssescalarmode> 7 "vsib_mem_operator" 21643 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19627 UNSPEC_GATHER) 21651 UNSPEC_GATHER)
19628 (parallel [(const_int 0) (const_int 1) 21652 (parallel [(const_int 0) (const_int 1)
19629 (const_int 2) (const_int 3)]))) 21653 (const_int 2) (const_int 3)])))
19630 (clobber (match_scratch:VI4F_256 1 "=&x"))] 21654 (clobber (match_scratch:VI4F_256 1 "=&x"))]
19631 "TARGET_AVX2" 21655 "TARGET_AVX2"
19632 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}" 21656 "%M3v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}"
19633 [(set_attr "type" "ssemov") 21657 [(set_attr "type" "ssemov")
19634 (set_attr "prefix" "vex") 21658 (set_attr "prefix" "vex")
19635 (set_attr "mode" "<sseinsnmode>")]) 21659 (set_attr "mode" "<sseinsnmode>")])
19636 21660
19637 (define_insn "*avx2_gatherdi<mode>_4" 21661 (define_insn "*avx2_gatherdi<VI4F_256:mode>_4"
19638 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x") 21662 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
19639 (vec_select:<VEC_GATHER_SRCDI> 21663 (vec_select:<VEC_GATHER_SRCDI>
19640 (unspec:VI4F_256 21664 (unspec:VI4F_256
19641 [(pc) 21665 [(pc)
19642 (match_operator:<ssescalarmode> 6 "vsib_mem_operator" 21666 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19650 UNSPEC_GATHER) 21674 UNSPEC_GATHER)
19651 (parallel [(const_int 0) (const_int 1) 21675 (parallel [(const_int 0) (const_int 1)
19652 (const_int 2) (const_int 3)]))) 21676 (const_int 2) (const_int 3)])))
19653 (clobber (match_scratch:VI4F_256 1 "=&x"))] 21677 (clobber (match_scratch:VI4F_256 1 "=&x"))]
19654 "TARGET_AVX2" 21678 "TARGET_AVX2"
19655 "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}" 21679 "%M2v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"
19656 [(set_attr "type" "ssemov") 21680 [(set_attr "type" "ssemov")
19657 (set_attr "prefix" "vex") 21681 (set_attr "prefix" "vex")
19658 (set_attr "mode" "<sseinsnmode>")]) 21682 (set_attr "mode" "<sseinsnmode>")])
19659
19660 ;; Memory operand override for -masm=intel of the v*gatherq* patterns.
19661 (define_mode_attr gatherq_mode
19662 [(V4SI "q") (V2DI "x") (V4SF "q") (V2DF "x")
19663 (V8SI "x") (V4DI "t") (V8SF "x") (V4DF "t")
19664 (V16SI "t") (V8DI "g") (V16SF "t") (V8DF "g")])
19665 21683
19666 (define_expand "<avx512>_gathersi<mode>" 21684 (define_expand "<avx512>_gathersi<mode>"
19667 [(parallel [(set (match_operand:VI48F 0 "register_operand") 21685 [(parallel [(set (match_operand:VI48F 0 "register_operand")
19668 (unspec:VI48F 21686 (unspec:VI48F
19669 [(match_operand:VI48F 1 "register_operand") 21687 [(match_operand:VI48F 1 "register_operand")
19680 operands[6] 21698 operands[6]
19681 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], 21699 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19682 operands[5]), UNSPEC_VSIBADDR); 21700 operands[5]), UNSPEC_VSIBADDR);
19683 }) 21701 })
19684 21702
19685 (define_insn "*avx512f_gathersi<mode>" 21703 (define_insn "*avx512f_gathersi<VI48F:mode>"
19686 [(set (match_operand:VI48F 0 "register_operand" "=&v") 21704 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19687 (unspec:VI48F 21705 (unspec:VI48F
19688 [(match_operand:VI48F 1 "register_operand" "0") 21706 [(match_operand:VI48F 1 "register_operand" "0")
19689 (match_operand:<avx512fmaskmode> 7 "register_operand" "2") 21707 (match_operand:<avx512fmaskmode> 7 "register_operand" "2")
19690 (match_operator:<ssescalarmode> 6 "vsib_mem_operator" 21708 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19694 (match_operand:SI 5 "const1248_operand" "n")] 21712 (match_operand:SI 5 "const1248_operand" "n")]
19695 UNSPEC_VSIBADDR)])] 21713 UNSPEC_VSIBADDR)])]
19696 UNSPEC_GATHER)) 21714 UNSPEC_GATHER))
19697 (clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))] 21715 (clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))]
19698 "TARGET_AVX512F" 21716 "TARGET_AVX512F"
19699 "v<sseintprefix>gatherd<ssemodesuffix>\t{%6, %0%{%2%}|%0%{%2%}, %<xtg_mode>6}" 21717 ;; %X6 so that we don't emit any *WORD PTR for -masm=intel, as
21718 ;; gas changed what it requires incompatibly.
21719 "%M4v<sseintprefix>gatherd<ssemodesuffix>\t{%6, %0%{%2%}|%0%{%2%}, %X6}"
19700 [(set_attr "type" "ssemov") 21720 [(set_attr "type" "ssemov")
19701 (set_attr "prefix" "evex") 21721 (set_attr "prefix" "evex")
19702 (set_attr "mode" "<sseinsnmode>")]) 21722 (set_attr "mode" "<sseinsnmode>")])
19703 21723
19704 (define_insn "*avx512f_gathersi<mode>_2" 21724 (define_insn "*avx512f_gathersi<VI48F:mode>_2"
19705 [(set (match_operand:VI48F 0 "register_operand" "=&v") 21725 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19706 (unspec:VI48F 21726 (unspec:VI48F
19707 [(pc) 21727 [(pc)
19708 (match_operand:<avx512fmaskmode> 6 "register_operand" "1") 21728 (match_operand:<avx512fmaskmode> 6 "register_operand" "1")
19709 (match_operator:<ssescalarmode> 5 "vsib_mem_operator" 21729 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
19713 (match_operand:SI 4 "const1248_operand" "n")] 21733 (match_operand:SI 4 "const1248_operand" "n")]
19714 UNSPEC_VSIBADDR)])] 21734 UNSPEC_VSIBADDR)])]
19715 UNSPEC_GATHER)) 21735 UNSPEC_GATHER))
19716 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))] 21736 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
19717 "TARGET_AVX512F" 21737 "TARGET_AVX512F"
19718 "v<sseintprefix>gatherd<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %<xtg_mode>5}" 21738 ;; %X5 so that we don't emit any *WORD PTR for -masm=intel, as
21739 ;; gas changed what it requires incompatibly.
21740 "%M3v<sseintprefix>gatherd<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %X5}"
19719 [(set_attr "type" "ssemov") 21741 [(set_attr "type" "ssemov")
19720 (set_attr "prefix" "evex") 21742 (set_attr "prefix" "evex")
19721 (set_attr "mode" "<sseinsnmode>")]) 21743 (set_attr "mode" "<sseinsnmode>")])
19722 21744
19723 21745
19738 operands[6] 21760 operands[6]
19739 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3], 21761 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19740 operands[5]), UNSPEC_VSIBADDR); 21762 operands[5]), UNSPEC_VSIBADDR);
19741 }) 21763 })
19742 21764
19743 (define_insn "*avx512f_gatherdi<mode>" 21765 (define_insn "*avx512f_gatherdi<VI48F:mode>"
19744 [(set (match_operand:VI48F 0 "register_operand" "=&v") 21766 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19745 (unspec:VI48F 21767 (unspec:VI48F
19746 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "0") 21768 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "0")
19747 (match_operand:QI 7 "register_operand" "2") 21769 (match_operand:QI 7 "register_operand" "2")
19748 (match_operator:<ssescalarmode> 6 "vsib_mem_operator" 21770 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19752 (match_operand:SI 5 "const1248_operand" "n")] 21774 (match_operand:SI 5 "const1248_operand" "n")]
19753 UNSPEC_VSIBADDR)])] 21775 UNSPEC_VSIBADDR)])]
19754 UNSPEC_GATHER)) 21776 UNSPEC_GATHER))
19755 (clobber (match_scratch:QI 2 "=&Yk"))] 21777 (clobber (match_scratch:QI 2 "=&Yk"))]
19756 "TARGET_AVX512F" 21778 "TARGET_AVX512F"
19757 { 21779 ;; %X6 so that we don't emit any *WORD PTR for -masm=intel, as
19758 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%6, %1%{%2%}|%1%{%2%}, %<gatherq_mode>6}"; 21780 ;; gas changed what it requires incompatibly.
19759 } 21781 "%M4v<sseintprefix>gatherq<ssemodesuffix>\t{%6, %1%{%2%}|%1%{%2%}, %X6}"
19760 [(set_attr "type" "ssemov") 21782 [(set_attr "type" "ssemov")
19761 (set_attr "prefix" "evex") 21783 (set_attr "prefix" "evex")
19762 (set_attr "mode" "<sseinsnmode>")]) 21784 (set_attr "mode" "<sseinsnmode>")])
19763 21785
19764 (define_insn "*avx512f_gatherdi<mode>_2" 21786 (define_insn "*avx512f_gatherdi<VI48F:mode>_2"
19765 [(set (match_operand:VI48F 0 "register_operand" "=&v") 21787 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19766 (unspec:VI48F 21788 (unspec:VI48F
19767 [(pc) 21789 [(pc)
19768 (match_operand:QI 6 "register_operand" "1") 21790 (match_operand:QI 6 "register_operand" "1")
19769 (match_operator:<ssescalarmode> 5 "vsib_mem_operator" 21791 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
19774 UNSPEC_VSIBADDR)])] 21796 UNSPEC_VSIBADDR)])]
19775 UNSPEC_GATHER)) 21797 UNSPEC_GATHER))
19776 (clobber (match_scratch:QI 1 "=&Yk"))] 21798 (clobber (match_scratch:QI 1 "=&Yk"))]
19777 "TARGET_AVX512F" 21799 "TARGET_AVX512F"
19778 { 21800 {
19779 if (<MODE>mode != <VEC_GATHER_SRCDI>mode) 21801 /* %X5 so that we don't emit any *WORD PTR for -masm=intel, as
21802 gas changed what it requires incompatibly. */
21803 if (<VI48F:MODE>mode != <VEC_GATHER_SRCDI>mode)
19780 { 21804 {
19781 if (<MODE_SIZE> != 64) 21805 if (<VI48F:MODE_SIZE> != 64)
19782 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %x0%{%1%}|%x0%{%1%}, %<gatherq_mode>5}"; 21806 return "%M3v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %x0%{%1%}|%x0%{%1%}, %X5}";
19783 else 21807 else
19784 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %t0%{%1%}|%t0%{%1%}, %t5}"; 21808 return "%M3v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %t0%{%1%}|%t0%{%1%}, %X5}";
19785 } 21809 }
19786 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %<gatherq_mode>5}"; 21810 return "%M3v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %X5}";
19787 } 21811 }
19788 [(set_attr "type" "ssemov") 21812 [(set_attr "type" "ssemov")
19789 (set_attr "prefix" "evex") 21813 (set_attr "prefix" "evex")
19790 (set_attr "mode" "<sseinsnmode>")]) 21814 (set_attr "mode" "<sseinsnmode>")])
19791 21815
19805 operands[5] 21829 operands[5]
19806 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2], 21830 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
19807 operands[4]), UNSPEC_VSIBADDR); 21831 operands[4]), UNSPEC_VSIBADDR);
19808 }) 21832 })
19809 21833
19810 (define_insn "*avx512f_scattersi<mode>" 21834 (define_insn "*avx512f_scattersi<VI48F:mode>"
19811 [(set (match_operator:VI48F 5 "vsib_mem_operator" 21835 [(set (match_operator:VI48F 5 "vsib_mem_operator"
19812 [(unspec:P 21836 [(unspec:P
19813 [(match_operand:P 0 "vsib_address_operand" "Tv") 21837 [(match_operand:P 0 "vsib_address_operand" "Tv")
19814 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v") 21838 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
19815 (match_operand:SI 4 "const1248_operand" "n")] 21839 (match_operand:SI 4 "const1248_operand" "n")]
19818 [(match_operand:<avx512fmaskmode> 6 "register_operand" "1") 21842 [(match_operand:<avx512fmaskmode> 6 "register_operand" "1")
19819 (match_operand:VI48F 3 "register_operand" "v")] 21843 (match_operand:VI48F 3 "register_operand" "v")]
19820 UNSPEC_SCATTER)) 21844 UNSPEC_SCATTER))
19821 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))] 21845 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
19822 "TARGET_AVX512F" 21846 "TARGET_AVX512F"
19823 "v<sseintprefix>scatterd<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}" 21847 ;; %X5 so that we don't emit any *WORD PTR for -masm=intel, as
21848 ;; gas changed what it requires incompatibly.
21849 "%M0v<sseintprefix>scatterd<ssemodesuffix>\t{%3, %5%{%1%}|%X5%{%1%}, %3}"
19824 [(set_attr "type" "ssemov") 21850 [(set_attr "type" "ssemov")
19825 (set_attr "prefix" "evex") 21851 (set_attr "prefix" "evex")
19826 (set_attr "mode" "<sseinsnmode>")]) 21852 (set_attr "mode" "<sseinsnmode>")])
19827 21853
19828 (define_expand "<avx512>_scatterdi<mode>" 21854 (define_expand "<avx512>_scatterdi<mode>"
19841 operands[5] 21867 operands[5]
19842 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2], 21868 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
19843 operands[4]), UNSPEC_VSIBADDR); 21869 operands[4]), UNSPEC_VSIBADDR);
19844 }) 21870 })
19845 21871
19846 (define_insn "*avx512f_scatterdi<mode>" 21872 (define_insn "*avx512f_scatterdi<VI48F:mode>"
19847 [(set (match_operator:VI48F 5 "vsib_mem_operator" 21873 [(set (match_operator:VI48F 5 "vsib_mem_operator"
19848 [(unspec:P 21874 [(unspec:P
19849 [(match_operand:P 0 "vsib_address_operand" "Tv") 21875 [(match_operand:P 0 "vsib_address_operand" "Tv")
19850 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v") 21876 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
19851 (match_operand:SI 4 "const1248_operand" "n")] 21877 (match_operand:SI 4 "const1248_operand" "n")]
19854 [(match_operand:QI 6 "register_operand" "1") 21880 [(match_operand:QI 6 "register_operand" "1")
19855 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand" "v")] 21881 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand" "v")]
19856 UNSPEC_SCATTER)) 21882 UNSPEC_SCATTER))
19857 (clobber (match_scratch:QI 1 "=&Yk"))] 21883 (clobber (match_scratch:QI 1 "=&Yk"))]
19858 "TARGET_AVX512F" 21884 "TARGET_AVX512F"
19859 { 21885 ;; %X5 so that we don't emit any *WORD PTR for -masm=intel, as
19860 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 8) 21886 ;; gas changed what it requires incompatibly.
19861 return "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"; 21887 "%M0v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%X5%{%1%}, %3}"
19862 return "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%t5%{%1%}, %3}";
19863 }
19864 [(set_attr "type" "ssemov") 21888 [(set_attr "type" "ssemov")
19865 (set_attr "prefix" "evex") 21889 (set_attr "prefix" "evex")
19866 (set_attr "mode" "<sseinsnmode>")]) 21890 (set_attr "mode" "<sseinsnmode>")])
19867 21891
19868 (define_insn "<avx512>_compress<mode>_mask" 21892 (define_insn "<avx512>_compress<mode>_mask"
19995 [(set_attr "type" "sse") 22019 [(set_attr "type" "sse")
19996 (set_attr "prefix" "evex") 22020 (set_attr "prefix" "evex")
19997 (set_attr "mode" "<MODE>")]) 22021 (set_attr "mode" "<MODE>")])
19998 22022
19999 (define_insn "avx512dq_fpclass<mode><mask_scalar_merge_name>" 22023 (define_insn "avx512dq_fpclass<mode><mask_scalar_merge_name>"
20000 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 22024 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
20001 (unspec:<avx512fmaskmode> 22025 (unspec:<avx512fmaskmode>
20002 [(match_operand:VF_AVX512VL 1 "register_operand" "v") 22026 [(match_operand:VF_AVX512VL 1 "vector_operand" "vm")
20003 (match_operand:QI 2 "const_0_to_255_operand" "n")] 22027 (match_operand:QI 2 "const_0_to_255_operand" "n")]
20004 UNSPEC_FPCLASS))] 22028 UNSPEC_FPCLASS))]
20005 "TARGET_AVX512DQ" 22029 "TARGET_AVX512DQ"
20006 "vfpclass<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"; 22030 "vfpclass<ssemodesuffix><vecmemsuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}";
20007 [(set_attr "type" "sse") 22031 [(set_attr "type" "sse")
20008 (set_attr "length_immediate" "1") 22032 (set_attr "length_immediate" "1")
20009 (set_attr "prefix" "evex") 22033 (set_attr "prefix" "evex")
20010 (set_attr "mode" "<MODE>")]) 22034 (set_attr "mode" "<MODE>")])
20011 22035
20012 (define_insn "avx512dq_vmfpclass<mode>" 22036 (define_insn "avx512dq_vmfpclass<mode><mask_scalar_merge_name>"
20013 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 22037 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
20014 (and:<avx512fmaskmode> 22038 (and:<avx512fmaskmode>
20015 (unspec:<avx512fmaskmode> 22039 (unspec:<avx512fmaskmode>
20016 [(match_operand:VF_128 1 "register_operand" "v") 22040 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")
20017 (match_operand:QI 2 "const_0_to_255_operand" "n")] 22041 (match_operand:QI 2 "const_0_to_255_operand" "n")]
20018 UNSPEC_FPCLASS) 22042 UNSPEC_FPCLASS)
20019 (const_int 1)))] 22043 (const_int 1)))]
20020 "TARGET_AVX512DQ" 22044 "TARGET_AVX512DQ"
20021 "vfpclass<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"; 22045 "vfpclass<ssescalarmodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}";
20022 [(set_attr "type" "sse") 22046 [(set_attr "type" "sse")
20023 (set_attr "length_immediate" "1") 22047 (set_attr "length_immediate" "1")
20024 (set_attr "prefix" "evex") 22048 (set_attr "prefix" "evex")
20025 (set_attr "mode" "<MODE>")]) 22049 (set_attr "mode" "<MODE>")])
20026 22050
20168 (set_attr "length_immediate" "1") 22192 (set_attr "length_immediate" "1")
20169 (set_attr "mode" "TI")]) 22193 (set_attr "mode" "TI")])
20170 22194
20171 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_<castmode>" 22195 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_<castmode>"
20172 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m") 22196 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
20173 (unspec:AVX512MODE2P 22197 (vec_concat:AVX512MODE2P
20174 [(match_operand:<ssequartermode> 1 "nonimmediate_operand" "xm,x")] 22198 (vec_concat:<ssehalfvecmode>
20175 UNSPEC_CAST))] 22199 (match_operand:<ssequartermode> 1 "nonimmediate_operand" "xm,x")
22200 (unspec:<ssequartermode> [(const_int 0)] UNSPEC_CAST))
22201 (unspec:<ssehalfvecmode> [(const_int 0)] UNSPEC_CAST)))]
20176 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))" 22202 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
20177 "#" 22203 "#"
20178 "&& reload_completed" 22204 "&& reload_completed"
20179 [(set (match_dup 0) (match_dup 1))] 22205 [(set (match_dup 0) (match_dup 1))]
20180 { 22206 {
20185 <ssequartermode>mode); 22211 <ssequartermode>mode);
20186 }) 22212 })
20187 22213
20188 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_256<castmode>" 22214 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_256<castmode>"
20189 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m") 22215 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
20190 (unspec:AVX512MODE2P 22216 (vec_concat:AVX512MODE2P
20191 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")] 22217 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")
20192 UNSPEC_CAST))] 22218 (unspec:<ssehalfvecmode> [(const_int 0)] UNSPEC_CAST)))]
20193 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))" 22219 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
20194 "#" 22220 "#"
20195 "&& reload_completed" 22221 "&& reload_completed"
20196 [(set (match_dup 0) (match_dup 1))] 22222 [(set (match_dup 0) (match_dup 1))]
20197 { 22223 {
20569 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")))] 22595 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")))]
20570 "TARGET_AVX512VPOPCNTDQ" 22596 "TARGET_AVX512VPOPCNTDQ"
20571 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}") 22597 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}")
20572 22598
20573 ;; Save multiple registers out-of-line. 22599 ;; Save multiple registers out-of-line.
20574 (define_insn "save_multiple<mode>" 22600 (define_insn "*save_multiple<mode>"
20575 [(match_parallel 0 "save_multiple" 22601 [(match_parallel 0 "save_multiple"
20576 [(use (match_operand:P 1 "symbol_operand"))])] 22602 [(use (match_operand:P 1 "symbol_operand"))])]
20577 "TARGET_SSE && TARGET_64BIT" 22603 "TARGET_SSE && TARGET_64BIT"
20578 "call\t%P1") 22604 "call\t%P1")
20579 22605
20580 ;; Restore multiple registers out-of-line. 22606 ;; Restore multiple registers out-of-line.
20581 (define_insn "restore_multiple<mode>" 22607 (define_insn "*restore_multiple<mode>"
20582 [(match_parallel 0 "restore_multiple" 22608 [(match_parallel 0 "restore_multiple"
20583 [(use (match_operand:P 1 "symbol_operand"))])] 22609 [(use (match_operand:P 1 "symbol_operand"))])]
20584 "TARGET_SSE && TARGET_64BIT" 22610 "TARGET_SSE && TARGET_64BIT"
20585 "call\t%P1") 22611 "call\t%P1")
20586 22612
20587 ;; Restore multiple registers out-of-line and return. 22613 ;; Restore multiple registers out-of-line and return.
20588 (define_insn "restore_multiple_and_return<mode>" 22614 (define_insn "*restore_multiple_and_return<mode>"
20589 [(match_parallel 0 "restore_multiple" 22615 [(match_parallel 0 "restore_multiple"
20590 [(return) 22616 [(return)
20591 (use (match_operand:P 1 "symbol_operand")) 22617 (use (match_operand:P 1 "symbol_operand"))
20592 (set (reg:DI SP_REG) (reg:DI R10_REG)) 22618 (set (reg:DI SP_REG) (reg:DI R10_REG))
20593 ])] 22619 ])]
20594 "TARGET_SSE && TARGET_64BIT" 22620 "TARGET_SSE && TARGET_64BIT"
20595 "jmp\t%P1") 22621 "jmp\t%P1")
20596 22622
20597 ;; Restore multiple registers out-of-line when hard frame pointer is used, 22623 ;; Restore multiple registers out-of-line when hard frame pointer is used,
20598 ;; perform the leave operation prior to returning (from the function). 22624 ;; perform the leave operation prior to returning (from the function).
20599 (define_insn "restore_multiple_leave_return<mode>" 22625 (define_insn "*restore_multiple_leave_return<mode>"
20600 [(match_parallel 0 "restore_multiple" 22626 [(match_parallel 0 "restore_multiple"
20601 [(return) 22627 [(return)
20602 (use (match_operand:P 1 "symbol_operand")) 22628 (use (match_operand:P 1 "symbol_operand"))
20603 (set (reg:DI SP_REG) (plus:DI (reg:DI BP_REG) (const_int 8))) 22629 (set (reg:DI SP_REG) (plus:DI (reg:DI BP_REG) (const_int 8)))
20604 (set (reg:DI BP_REG) (mem:DI (reg:DI BP_REG))) 22630 (set (reg:DI BP_REG) (mem:DI (reg:DI BP_REG)))
20613 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "vm")))] 22639 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "vm")))]
20614 "TARGET_AVX512BITALG" 22640 "TARGET_AVX512BITALG"
20615 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}") 22641 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}")
20616 22642
20617 (define_insn "vgf2p8affineinvqb_<mode><mask_name>" 22643 (define_insn "vgf2p8affineinvqb_<mode><mask_name>"
20618 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v") 22644 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,v")
20619 (unspec:VI1_AVX512F 22645 (unspec:VI1_AVX512F
20620 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v") 22646 [(match_operand:VI1_AVX512F 1 "register_operand" "0,v")
20621 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm") 22647 (match_operand:VI1_AVX512F 2 "vector_operand" "xBm,vm")
20622 (match_operand:QI 3 "const_0_to_255_operand" "n,n,n")] 22648 (match_operand:QI 3 "const_0_to_255_operand" "n,n")]
20623 UNSPEC_GF2P8AFFINEINV))] 22649 UNSPEC_GF2P8AFFINEINV))]
20624 "TARGET_GFNI" 22650 "TARGET_GFNI"
20625 "@ 22651 "@
20626 gf2p8affineinvqb\t{%3, %2, %0| %0, %2, %3} 22652 gf2p8affineinvqb\t{%3, %2, %0| %0, %2, %3}
20627 vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}
20628 vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}" 22653 vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}"
20629 [(set_attr "isa" "noavx,avx,avx512f") 22654 [(set_attr "isa" "noavx,avx")
20630 (set_attr "prefix_data16" "1,*,*") 22655 (set_attr "prefix_data16" "1,*")
20631 (set_attr "prefix_extra" "1") 22656 (set_attr "prefix_extra" "1")
20632 (set_attr "prefix" "orig,maybe_evex,evex") 22657 (set_attr "prefix" "orig,maybe_evex")
20633 (set_attr "mode" "<sseinsnmode>")]) 22658 (set_attr "mode" "<sseinsnmode>")])
20634 22659
20635 (define_insn "vgf2p8affineqb_<mode><mask_name>" 22660 (define_insn "vgf2p8affineqb_<mode><mask_name>"
20636 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v") 22661 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,v")
20637 (unspec:VI1_AVX512F 22662 (unspec:VI1_AVX512F
20638 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v") 22663 [(match_operand:VI1_AVX512F 1 "register_operand" "0,v")
20639 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm") 22664 (match_operand:VI1_AVX512F 2 "vector_operand" "xBm,vm")
20640 (match_operand:QI 3 "const_0_to_255_operand" "n,n,n")] 22665 (match_operand:QI 3 "const_0_to_255_operand" "n,n")]
20641 UNSPEC_GF2P8AFFINE))] 22666 UNSPEC_GF2P8AFFINE))]
20642 "TARGET_GFNI" 22667 "TARGET_GFNI"
20643 "@ 22668 "@
20644 gf2p8affineqb\t{%3, %2, %0| %0, %2, %3} 22669 gf2p8affineqb\t{%3, %2, %0| %0, %2, %3}
20645 vgf2p8affineqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}
20646 vgf2p8affineqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}" 22670 vgf2p8affineqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}"
20647 [(set_attr "isa" "noavx,avx,avx512f") 22671 [(set_attr "isa" "noavx,avx")
20648 (set_attr "prefix_data16" "1,*,*") 22672 (set_attr "prefix_data16" "1,*")
20649 (set_attr "prefix_extra" "1") 22673 (set_attr "prefix_extra" "1")
20650 (set_attr "prefix" "orig,maybe_evex,evex") 22674 (set_attr "prefix" "orig,maybe_evex")
20651 (set_attr "mode" "<sseinsnmode>")]) 22675 (set_attr "mode" "<sseinsnmode>")])
20652 22676
20653 (define_insn "vgf2p8mulb_<mode><mask_name>" 22677 (define_insn "vgf2p8mulb_<mode><mask_name>"
20654 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v") 22678 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,v")
20655 (unspec:VI1_AVX512F 22679 (unspec:VI1_AVX512F
20656 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v") 22680 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,v")
20657 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm")] 22681 (match_operand:VI1_AVX512F 2 "vector_operand" "xBm,vm")]
20658 UNSPEC_GF2P8MUL))] 22682 UNSPEC_GF2P8MUL))]
20659 "TARGET_GFNI" 22683 "TARGET_GFNI"
20660 "@ 22684 "@
20661 gf2p8mulb\t{%2, %0| %0, %2} 22685 gf2p8mulb\t{%2, %0| %0, %2}
20662 vgf2p8mulb\t{%2, %1, %0<mask_operand3>| %0<mask_operand3>, %1, %2}
20663 vgf2p8mulb\t{%2, %1, %0<mask_operand3>| %0<mask_operand3>, %1, %2}" 22686 vgf2p8mulb\t{%2, %1, %0<mask_operand3>| %0<mask_operand3>, %1, %2}"
20664 [(set_attr "isa" "noavx,avx,avx512f") 22687 [(set_attr "isa" "noavx,avx")
20665 (set_attr "prefix_data16" "1,*,*") 22688 (set_attr "prefix_data16" "1,*")
20666 (set_attr "prefix_extra" "1") 22689 (set_attr "prefix_extra" "1")
20667 (set_attr "prefix" "orig,maybe_evex,evex") 22690 (set_attr "prefix" "orig,maybe_evex")
20668 (set_attr "mode" "<sseinsnmode>")]) 22691 (set_attr "mode" "<sseinsnmode>")])
20669 22692
20670 (define_insn "vpshrd_<mode><mask_name>" 22693 (define_insn "vpshrd_<mode><mask_name>"
20671 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v") 22694 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20672 (unspec:VI248_AVX512VL 22695 (unspec:VI248_AVX512VL
21024 23047
21025 (define_insn "vaesdec_<mode>" 23048 (define_insn "vaesdec_<mode>"
21026 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v") 23049 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
21027 (unspec:VI1_AVX512VL_F 23050 (unspec:VI1_AVX512VL_F
21028 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v") 23051 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
21029 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "v")] 23052 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "vm")]
21030 UNSPEC_VAESDEC))] 23053 UNSPEC_VAESDEC))]
21031 "TARGET_VAES" 23054 "TARGET_VAES"
21032 "vaesdec\t{%2, %1, %0|%0, %1, %2}" 23055 "vaesdec\t{%2, %1, %0|%0, %1, %2}"
21033 ) 23056 )
21034 23057
21035 (define_insn "vaesdeclast_<mode>" 23058 (define_insn "vaesdeclast_<mode>"
21036 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v") 23059 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
21037 (unspec:VI1_AVX512VL_F 23060 (unspec:VI1_AVX512VL_F
21038 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v") 23061 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
21039 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "v")] 23062 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "vm")]
21040 UNSPEC_VAESDECLAST))] 23063 UNSPEC_VAESDECLAST))]
21041 "TARGET_VAES" 23064 "TARGET_VAES"
21042 "vaesdeclast\t{%2, %1, %0|%0, %1, %2}" 23065 "vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
21043 ) 23066 )
21044 23067
21071 "TARGET_VPCLMULQDQ" 23094 "TARGET_VPCLMULQDQ"
21072 "vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}" 23095 "vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
21073 [(set_attr "mode" "DI")]) 23096 [(set_attr "mode" "DI")])
21074 23097
21075 (define_insn "avx512vl_vpshufbitqmb<mode><mask_scalar_merge_name>" 23098 (define_insn "avx512vl_vpshufbitqmb<mode><mask_scalar_merge_name>"
21076 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk") 23099 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
21077 (unspec:<avx512fmaskmode> 23100 (unspec:<avx512fmaskmode>
21078 [(match_operand:VI1_AVX512VLBW 1 "register_operand" "v") 23101 [(match_operand:VI1_AVX512VLBW 1 "register_operand" "v")
21079 (match_operand:VI1_AVX512VLBW 2 "nonimmediate_operand" "vm")] 23102 (match_operand:VI1_AVX512VLBW 2 "nonimmediate_operand" "vm")]
21080 UNSPEC_VPSHUFBIT))] 23103 UNSPEC_VPSHUFBIT))]
21081 "TARGET_AVX512BITALG" 23104 "TARGET_AVX512BITALG"
21082 "vpshufbitqmb\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}" 23105 "vpshufbitqmb\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
21083 [(set_attr "prefix" "evex") 23106 [(set_attr "prefix" "evex")
21084 (set_attr "mode" "<sseinsnmode>")]) 23107 (set_attr "mode" "<sseinsnmode>")])
23108
23109 (define_mode_iterator VI48_AVX512VP2VL
23110 [V8DI
23111 (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
23112 (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")])
23113
23114 (define_insn "avx512vp2intersect_2intersect<mode>"
23115 [(set (match_operand:P2QI 0 "register_operand" "=k")
23116 (unspec:P2QI
23117 [(match_operand:VI48_AVX512VP2VL 1 "register_operand" "v")
23118 (match_operand:VI48_AVX512VP2VL 2 "vector_operand" "vm")]
23119 UNSPEC_VP2INTERSECT))]
23120 "TARGET_AVX512VP2INTERSECT"
23121 "vp2intersect<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
23122 [(set_attr ("prefix") ("evex"))])
23123
23124 (define_insn "avx512vp2intersect_2intersectv16si"
23125 [(set (match_operand:P2HI 0 "register_operand" "=k")
23126 (unspec:P2HI [(match_operand:V16SI 1 "register_operand" "v")
23127 (match_operand:V16SI 2 "vector_operand" "vm")]
23128 UNSPEC_VP2INTERSECT))]
23129 "TARGET_AVX512VP2INTERSECT"
23130 "vp2intersectd\t{%2, %1, %0|%0, %1, %2}"
23131 [(set_attr ("prefix") ("evex"))])
23132
23133 (define_mode_iterator BF16 [V32HI (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
23134 ;; Converting from BF to SF
23135 (define_mode_attr bf16_cvt_2sf
23136 [(V32HI "V16SF") (V16HI "V8SF") (V8HI "V4SF")])
23137 ;; Converting from SF to BF
23138 (define_mode_attr sf_cvt_bf16
23139 [(V4SF "V8HI") (V8SF "V8HI") (V16SF "V16HI")])
23140 ;; Mapping from BF to SF
23141 (define_mode_attr sf_bf16
23142 [(V4SF "V8HI") (V8SF "V16HI") (V16SF "V32HI")])
23143
23144 (define_expand "avx512f_cvtne2ps2bf16_<mode>_maskz"
23145 [(match_operand:BF16 0 "register_operand")
23146 (match_operand:<bf16_cvt_2sf> 1 "register_operand")
23147 (match_operand:<bf16_cvt_2sf> 2 "register_operand")
23148 (match_operand:<avx512fmaskmode> 3 "register_operand")]
23149 "TARGET_AVX512BF16"
23150 {
23151 emit_insn (gen_avx512f_cvtne2ps2bf16_<mode>_mask(operands[0], operands[1],
23152 operands[2], CONST0_RTX(<MODE>mode), operands[3]));
23153 DONE;
23154 })
23155
23156 (define_insn "avx512f_cvtne2ps2bf16_<mode><mask_name>"
23157 [(set (match_operand:BF16 0 "register_operand" "=v")
23158 (unspec:BF16
23159 [(match_operand:<bf16_cvt_2sf> 1 "register_operand" "v")
23160 (match_operand:<bf16_cvt_2sf> 2 "register_operand" "v")]
23161 UNSPEC_VCVTNE2PS2BF16))]
23162 "TARGET_AVX512BF16"
23163 "vcvtne2ps2bf16\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}")
23164
23165 (define_expand "avx512f_cvtneps2bf16_<mode>_maskz"
23166 [(match_operand:<sf_cvt_bf16> 0 "register_operand")
23167 (match_operand:VF1_AVX512VL 1 "register_operand")
23168 (match_operand:<avx512fmaskmode> 2 "register_operand")]
23169 "TARGET_AVX512BF16"
23170 {
23171 emit_insn (gen_avx512f_cvtneps2bf16_<mode>_mask(operands[0], operands[1],
23172 CONST0_RTX(<sf_cvt_bf16>mode), operands[2]));
23173 DONE;
23174 })
23175
23176 (define_insn "avx512f_cvtneps2bf16_<mode><mask_name>"
23177 [(set (match_operand:<sf_cvt_bf16> 0 "register_operand" "=v")
23178 (unspec:<sf_cvt_bf16>
23179 [(match_operand:VF1_AVX512VL 1 "register_operand" "v")]
23180 UNSPEC_VCVTNEPS2BF16))]
23181 "TARGET_AVX512BF16"
23182 "vcvtneps2bf16\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}")
23183
23184 (define_expand "avx512f_dpbf16ps_<mode>_maskz"
23185 [(match_operand:VF1_AVX512VL 0 "register_operand")
23186 (match_operand:VF1_AVX512VL 1 "register_operand")
23187 (match_operand:<sf_bf16> 2 "register_operand")
23188 (match_operand:<sf_bf16> 3 "register_operand")
23189 (match_operand:<avx512fmaskhalfmode> 4 "register_operand")]
23190 "TARGET_AVX512BF16"
23191 {
23192 emit_insn (gen_avx512f_dpbf16ps_<mode>_maskz_1(operands[0], operands[1],
23193 operands[2], operands[3], CONST0_RTX(<MODE>mode), operands[4]));
23194 DONE;
23195 })
23196
23197 (define_insn "avx512f_dpbf16ps_<mode><maskz_half_name>"
23198 [(set (match_operand:VF1_AVX512VL 0 "register_operand" "=v")
23199 (unspec:VF1_AVX512VL
23200 [(match_operand:VF1_AVX512VL 1 "register_operand" "0")
23201 (match_operand:<sf_bf16> 2 "register_operand" "v")
23202 (match_operand:<sf_bf16> 3 "register_operand" "v")]
23203 UNSPEC_VDPBF16PS))]
23204 "TARGET_AVX512BF16"
23205 "vdpbf16ps\t{%3, %2, %0<maskz_half_operand4>|%0<maskz_half_operand4>, %2, %3}")
23206
23207 (define_insn "avx512f_dpbf16ps_<mode>_mask"
23208 [(set (match_operand:VF1_AVX512VL 0 "register_operand" "=v")
23209 (vec_merge:VF1_AVX512VL
23210 (unspec:VF1_AVX512VL
23211 [(match_operand:VF1_AVX512VL 1 "register_operand" "0")
23212 (match_operand:<sf_bf16> 2 "register_operand" "v")
23213 (match_operand:<sf_bf16> 3 "register_operand" "v")]
23214 UNSPEC_VDPBF16PS)
23215 (match_dup 1)
23216 (match_operand:<avx512fmaskhalfmode> 4 "register_operand" "Yk")))]
23217 "TARGET_AVX512BF16"
23218 "vdpbf16ps\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3}")