comparison gcc/internal-fn.c @ 145:1830386684a0

gcc-9.2.0
author anatofuz
date Thu, 13 Feb 2020 11:34:05 +0900
parents 84e7813d76e9
children
comparison
equal deleted inserted replaced
131:84e7813d76e9 145:1830386684a0
1 /* Internal functions. 1 /* Internal functions.
2 Copyright (C) 2011-2018 Free Software Foundation, Inc. 2 Copyright (C) 2011-2020 Free Software Foundation, Inc.
3 3
4 This file is part of GCC. 4 This file is part of GCC.
5 5
6 GCC is free software; you can redistribute it and/or modify it under 6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free 7 the terms of the GNU General Public License as published by the Free
101 direct_internal_fn. */ 101 direct_internal_fn. */
102 #define not_direct { -2, -2, false } 102 #define not_direct { -2, -2, false }
103 #define mask_load_direct { -1, 2, false } 103 #define mask_load_direct { -1, 2, false }
104 #define load_lanes_direct { -1, -1, false } 104 #define load_lanes_direct { -1, -1, false }
105 #define mask_load_lanes_direct { -1, -1, false } 105 #define mask_load_lanes_direct { -1, -1, false }
106 #define gather_load_direct { -1, -1, false } 106 #define gather_load_direct { 3, 1, false }
107 #define mask_store_direct { 3, 2, false } 107 #define mask_store_direct { 3, 2, false }
108 #define store_lanes_direct { 0, 0, false } 108 #define store_lanes_direct { 0, 0, false }
109 #define mask_store_lanes_direct { 0, 0, false } 109 #define mask_store_lanes_direct { 0, 0, false }
110 #define scatter_store_direct { 3, 3, false } 110 #define scatter_store_direct { 3, 1, false }
111 #define unary_direct { 0, 0, true } 111 #define unary_direct { 0, 0, true }
112 #define binary_direct { 0, 0, true } 112 #define binary_direct { 0, 0, true }
113 #define ternary_direct { 0, 0, true } 113 #define ternary_direct { 0, 0, true }
114 #define cond_unary_direct { 1, 1, true } 114 #define cond_unary_direct { 1, 1, true }
115 #define cond_binary_direct { 1, 1, true } 115 #define cond_binary_direct { 1, 1, true }
116 #define cond_ternary_direct { 1, 1, true } 116 #define cond_ternary_direct { 1, 1, true }
117 #define while_direct { 0, 2, false } 117 #define while_direct { 0, 2, false }
118 #define fold_extract_direct { 2, 2, false } 118 #define fold_extract_direct { 2, 2, false }
119 #define fold_left_direct { 1, 1, false } 119 #define fold_left_direct { 1, 1, false }
120 #define mask_fold_left_direct { 1, 1, false }
121 #define check_ptrs_direct { 0, 0, false }
120 122
121 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = { 123 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
122 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct, 124 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
123 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct, 125 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
124 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \ 126 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
146 /* Expand LOAD_LANES call STMT using optab OPTAB. */ 148 /* Expand LOAD_LANES call STMT using optab OPTAB. */
147 149
148 static void 150 static void
149 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) 151 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
150 { 152 {
151 struct expand_operand ops[2]; 153 class expand_operand ops[2];
152 tree type, lhs, rhs; 154 tree type, lhs, rhs;
153 rtx target, mem; 155 rtx target, mem;
154 156
155 lhs = gimple_call_lhs (stmt); 157 lhs = gimple_call_lhs (stmt);
156 rhs = gimple_call_arg (stmt, 0); 158 rhs = gimple_call_arg (stmt, 0);
170 /* Expand STORE_LANES call STMT using optab OPTAB. */ 172 /* Expand STORE_LANES call STMT using optab OPTAB. */
171 173
172 static void 174 static void
173 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) 175 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
174 { 176 {
175 struct expand_operand ops[2]; 177 class expand_operand ops[2];
176 tree type, lhs, rhs; 178 tree type, lhs, rhs;
177 rtx target, reg; 179 rtx target, reg;
178 180
179 lhs = gimple_call_lhs (stmt); 181 lhs = gimple_call_lhs (stmt);
180 rhs = gimple_call_arg (stmt, 0); 182 rhs = gimple_call_arg (stmt, 0);
224 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 226 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
225 else 227 else
226 target = gen_reg_rtx (Pmode); 228 target = gen_reg_rtx (Pmode);
227 rtx size = expand_normal (gimple_call_arg (stmt, 0)); 229 rtx size = expand_normal (gimple_call_arg (stmt, 0));
228 rtx align = expand_normal (gimple_call_arg (stmt, 1)); 230 rtx align = expand_normal (gimple_call_arg (stmt, 1));
229 struct expand_operand ops[3]; 231 class expand_operand ops[3];
230 create_output_operand (&ops[0], target, Pmode); 232 create_output_operand (&ops[0], target, Pmode);
231 create_input_operand (&ops[1], size, Pmode); 233 create_input_operand (&ops[1], size, Pmode);
232 create_input_operand (&ops[2], align, Pmode); 234 create_input_operand (&ops[2], align, Pmode);
233 gcc_assert (targetm.have_omp_simt_enter ()); 235 gcc_assert (targetm.have_omp_simt_enter ());
234 expand_insn (targetm.code_for_omp_simt_enter, 3, ops); 236 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
239 static void 241 static void
240 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt) 242 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
241 { 243 {
242 gcc_checking_assert (!gimple_call_lhs (stmt)); 244 gcc_checking_assert (!gimple_call_lhs (stmt));
243 rtx arg = expand_normal (gimple_call_arg (stmt, 0)); 245 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
244 struct expand_operand ops[1]; 246 class expand_operand ops[1];
245 create_input_operand (&ops[0], arg, Pmode); 247 create_input_operand (&ops[0], arg, Pmode);
246 gcc_assert (targetm.have_omp_simt_exit ()); 248 gcc_assert (targetm.have_omp_simt_exit ());
247 expand_insn (targetm.code_for_omp_simt_exit, 1, ops); 249 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
248 } 250 }
249 251
282 return; 284 return;
283 285
284 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 286 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
285 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); 287 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
286 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); 288 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
287 struct expand_operand ops[2]; 289 class expand_operand ops[2];
288 create_output_operand (&ops[0], target, mode); 290 create_output_operand (&ops[0], target, mode);
289 create_input_operand (&ops[1], cond, mode); 291 create_input_operand (&ops[1], cond, mode);
290 gcc_assert (targetm.have_omp_simt_last_lane ()); 292 gcc_assert (targetm.have_omp_simt_last_lane ());
291 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops); 293 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
292 } 294 }
301 return; 303 return;
302 304
303 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 305 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
304 rtx ctr = expand_normal (gimple_call_arg (stmt, 0)); 306 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
305 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); 307 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
306 struct expand_operand ops[2]; 308 class expand_operand ops[2];
307 create_output_operand (&ops[0], target, mode); 309 create_output_operand (&ops[0], target, mode);
308 create_input_operand (&ops[1], ctr, mode); 310 create_input_operand (&ops[1], ctr, mode);
309 gcc_assert (targetm.have_omp_simt_ordered ()); 311 gcc_assert (targetm.have_omp_simt_ordered ());
310 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops); 312 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
311 } 313 }
321 return; 323 return;
322 324
323 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 325 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
324 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); 326 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
325 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); 327 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
326 struct expand_operand ops[2]; 328 class expand_operand ops[2];
327 create_output_operand (&ops[0], target, mode); 329 create_output_operand (&ops[0], target, mode);
328 create_input_operand (&ops[1], cond, mode); 330 create_input_operand (&ops[1], cond, mode);
329 gcc_assert (targetm.have_omp_simt_vote_any ()); 331 gcc_assert (targetm.have_omp_simt_vote_any ());
330 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops); 332 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
331 } 333 }
342 344
343 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 345 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
344 rtx src = expand_normal (gimple_call_arg (stmt, 0)); 346 rtx src = expand_normal (gimple_call_arg (stmt, 0));
345 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); 347 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
346 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); 348 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
347 struct expand_operand ops[3]; 349 class expand_operand ops[3];
348 create_output_operand (&ops[0], target, mode); 350 create_output_operand (&ops[0], target, mode);
349 create_input_operand (&ops[1], src, mode); 351 create_input_operand (&ops[1], src, mode);
350 create_input_operand (&ops[2], idx, SImode); 352 create_input_operand (&ops[2], idx, SImode);
351 gcc_assert (targetm.have_omp_simt_xchg_bfly ()); 353 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
352 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops); 354 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
363 365
364 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); 366 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
365 rtx src = expand_normal (gimple_call_arg (stmt, 0)); 367 rtx src = expand_normal (gimple_call_arg (stmt, 0));
366 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); 368 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
367 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); 369 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
368 struct expand_operand ops[3]; 370 class expand_operand ops[3];
369 create_output_operand (&ops[0], target, mode); 371 create_output_operand (&ops[0], target, mode);
370 create_input_operand (&ops[1], src, mode); 372 create_input_operand (&ops[1], src, mode);
371 create_input_operand (&ops[2], idx, SImode); 373 create_input_operand (&ops[2], idx, SImode);
372 gcc_assert (targetm.have_omp_simt_xchg_idx ()); 374 gcc_assert (targetm.have_omp_simt_xchg_idx ());
373 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops); 375 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
771 { 773 {
772 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab 774 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
773 : usubv4_optab, mode); 775 : usubv4_optab, mode);
774 if (icode != CODE_FOR_nothing) 776 if (icode != CODE_FOR_nothing)
775 { 777 {
776 struct expand_operand ops[4]; 778 class expand_operand ops[4];
777 rtx_insn *last = get_last_insn (); 779 rtx_insn *last = get_last_insn ();
778 780
779 res = gen_reg_rtx (mode); 781 res = gen_reg_rtx (mode);
780 create_output_operand (&ops[0], res, mode); 782 create_output_operand (&ops[0], res, mode);
781 create_input_operand (&ops[1], op0, mode); 783 create_input_operand (&ops[1], op0, mode);
992 { 994 {
993 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab 995 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
994 : subv4_optab, mode); 996 : subv4_optab, mode);
995 if (icode != CODE_FOR_nothing) 997 if (icode != CODE_FOR_nothing)
996 { 998 {
997 struct expand_operand ops[4]; 999 class expand_operand ops[4];
998 rtx_insn *last = get_last_insn (); 1000 rtx_insn *last = get_last_insn ();
999 1001
1000 res = gen_reg_rtx (mode); 1002 res = gen_reg_rtx (mode);
1001 create_output_operand (&ops[0], res, mode); 1003 create_output_operand (&ops[0], res, mode);
1002 create_input_operand (&ops[1], op0, mode); 1004 create_input_operand (&ops[1], op0, mode);
1143 } 1145 }
1144 1146
1145 enum insn_code icode = optab_handler (negv3_optab, mode); 1147 enum insn_code icode = optab_handler (negv3_optab, mode);
1146 if (icode != CODE_FOR_nothing) 1148 if (icode != CODE_FOR_nothing)
1147 { 1149 {
1148 struct expand_operand ops[3]; 1150 class expand_operand ops[3];
1149 rtx_insn *last = get_last_insn (); 1151 rtx_insn *last = get_last_insn ();
1150 1152
1151 res = gen_reg_rtx (mode); 1153 res = gen_reg_rtx (mode);
1152 create_output_operand (&ops[0], res, mode); 1154 create_output_operand (&ops[0], res, mode);
1153 create_input_operand (&ops[1], op1, mode); 1155 create_input_operand (&ops[1], op1, mode);
1405 } 1407 }
1406 1408
1407 /* s1 * s2 -> ur */ 1409 /* s1 * s2 -> ur */
1408 if (!uns0_p && !uns1_p && unsr_p) 1410 if (!uns0_p && !uns1_p && unsr_p)
1409 { 1411 {
1410 rtx tem, tem2; 1412 rtx tem;
1411 switch (pos_neg0 | pos_neg1) 1413 switch (pos_neg0 | pos_neg1)
1412 { 1414 {
1413 case 1: /* Both operands known to be non-negative. */ 1415 case 1: /* Both operands known to be non-negative. */
1414 goto do_main; 1416 goto do_main;
1415 case 2: /* Both operands known to be negative. */ 1417 case 2: /* Both operands known to be negative. */
1435 ops.op0 = make_tree (ops.type, op0); 1437 ops.op0 = make_tree (ops.type, op0);
1436 ops.op1 = make_tree (ops.type, op1); 1438 ops.op1 = make_tree (ops.type, op1);
1437 ops.op2 = NULL_TREE; 1439 ops.op2 = NULL_TREE;
1438 ops.location = loc; 1440 ops.location = loc;
1439 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); 1441 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1440 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false, 1442 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1441 OPTAB_LIB_WIDEN); 1443 true, mode, NULL_RTX, NULL, done_label,
1442 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
1443 NULL_RTX, NULL, done_label,
1444 profile_probability::very_likely ()); 1444 profile_probability::very_likely ());
1445 goto do_error_label; 1445 goto do_error_label;
1446 } 1446 }
1447 /* The general case, do all the needed comparisons at runtime. */ 1447 /* The general case, do all the needed comparisons at runtime. */
1448 rtx_code_label *do_main_label, *after_negate_label; 1448 rtx_code_label *do_main_label, *after_negate_label;
1469 the arguments. */ 1469 the arguments. */
1470 arg0 = error_mark_node; 1470 arg0 = error_mark_node;
1471 arg1 = error_mark_node; 1471 arg1 = error_mark_node;
1472 emit_jump (do_main_label); 1472 emit_jump (do_main_label);
1473 emit_label (after_negate_label); 1473 emit_label (after_negate_label);
1474 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false, 1474 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1475 OPTAB_LIB_WIDEN); 1475 OPTAB_LIB_WIDEN);
1476 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX, 1476 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1477 NULL, do_main_label, profile_probability::very_likely ()); 1477 NULL, do_main_label,
1478 profile_probability::very_likely ());
1478 /* One argument is negative here, the other positive. This 1479 /* One argument is negative here, the other positive. This
1479 overflows always, unless one of the arguments is 0. But 1480 overflows always, unless one of the arguments is 0. But
1480 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1 1481 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1481 is, thus we can keep do_main code oring in overflow as is. */ 1482 is, thus we can keep do_main code oring in overflow as is. */
1482 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX, 1483 if (pos_neg0 != 2)
1483 NULL, do_main_label, profile_probability::very_likely ()); 1484 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1485 NULL, do_main_label,
1486 profile_probability::very_unlikely ());
1487 if (pos_neg1 != 2)
1488 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1489 NULL, do_main_label,
1490 profile_probability::very_unlikely ());
1484 expand_arith_set_overflow (lhs, target); 1491 expand_arith_set_overflow (lhs, target);
1485 emit_label (do_main_label); 1492 emit_label (do_main_label);
1486 goto do_main; 1493 goto do_main;
1487 default: 1494 default:
1488 gcc_unreachable (); 1495 gcc_unreachable ();
1536 goto do_error_label; 1543 goto do_error_label;
1537 } 1544 }
1538 } 1545 }
1539 if (icode != CODE_FOR_nothing) 1546 if (icode != CODE_FOR_nothing)
1540 { 1547 {
1541 struct expand_operand ops[4]; 1548 class expand_operand ops[4];
1542 rtx_insn *last = get_last_insn (); 1549 rtx_insn *last = get_last_insn ();
1543 1550
1544 res = gen_reg_rtx (mode); 1551 res = gen_reg_rtx (mode);
1545 create_output_operand (&ops[0], res, mode); 1552 create_output_operand (&ops[0], res, mode);
1546 create_input_operand (&ops[1], op0, mode); 1553 create_input_operand (&ops[1], op0, mode);
1751 profile_probability::unlikely ()); 1758 profile_probability::unlikely ());
1752 1759
1753 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from 1760 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1754 hmode to mode, the multiplication will never overflow. We can 1761 hmode to mode, the multiplication will never overflow. We can
1755 do just one hmode x hmode => mode widening multiplication. */ 1762 do just one hmode x hmode => mode widening multiplication. */
1756 rtx lopart0s = lopart0, lopart1s = lopart1;
1757 if (GET_CODE (lopart0) == SUBREG)
1758 {
1759 lopart0s = shallow_copy_rtx (lopart0);
1760 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
1761 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1762 }
1763 if (GET_CODE (lopart1) == SUBREG)
1764 {
1765 lopart1s = shallow_copy_rtx (lopart1);
1766 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
1767 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1768 }
1769 tree halfstype = build_nonstandard_integer_type (hprec, uns); 1763 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1770 ops.op0 = make_tree (halfstype, lopart0s); 1764 ops.op0 = make_tree (halfstype, lopart0);
1771 ops.op1 = make_tree (halfstype, lopart1s); 1765 ops.op1 = make_tree (halfstype, lopart1);
1772 ops.code = WIDEN_MULT_EXPR; 1766 ops.code = WIDEN_MULT_EXPR;
1773 ops.type = type; 1767 ops.type = type;
1774 rtx thisres 1768 rtx thisres
1775 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); 1769 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1776 emit_move_insn (res, thisres); 1770 emit_move_insn (res, thisres);
2485 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */ 2479 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2486 2480
2487 static void 2481 static void
2488 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab) 2482 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2489 { 2483 {
2490 struct expand_operand ops[3]; 2484 class expand_operand ops[3];
2491 tree type, lhs, rhs, maskt; 2485 tree type, lhs, rhs, maskt;
2492 rtx mem, target, mask; 2486 rtx mem, target, mask;
2493 insn_code icode; 2487 insn_code icode;
2494 2488
2495 maskt = gimple_call_arg (stmt, 2); 2489 maskt = gimple_call_arg (stmt, 2);
2520 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */ 2514 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2521 2515
2522 static void 2516 static void
2523 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab) 2517 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2524 { 2518 {
2525 struct expand_operand ops[3]; 2519 class expand_operand ops[3];
2526 tree type, lhs, rhs, maskt; 2520 tree type, lhs, rhs, maskt;
2527 rtx mem, reg, mask; 2521 rtx mem, reg, mask;
2528 insn_code icode; 2522 insn_code icode;
2529 2523
2530 maskt = gimple_call_arg (stmt, 2); 2524 maskt = gimple_call_arg (stmt, 2);
2575 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function 2569 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2576 should never be called. */ 2570 should never be called. */
2577 2571
2578 static void 2572 static void
2579 expand_VA_ARG (internal_fn, gcall *) 2573 expand_VA_ARG (internal_fn, gcall *)
2574 {
2575 gcc_unreachable ();
2576 }
2577
2578 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2579 dummy function should never be called. */
2580
2581 static void
2582 expand_VEC_CONVERT (internal_fn, gcall *)
2580 { 2583 {
2581 gcc_unreachable (); 2584 gcc_unreachable ();
2582 } 2585 }
2583 2586
2584 /* Expand the IFN_UNIQUE function according to its first argument. */ 2587 /* Expand the IFN_UNIQUE function according to its first argument. */
2772 rtx base_rtx = expand_normal (base); 2775 rtx base_rtx = expand_normal (base);
2773 rtx offset_rtx = expand_normal (offset); 2776 rtx offset_rtx = expand_normal (offset);
2774 HOST_WIDE_INT scale_int = tree_to_shwi (scale); 2777 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
2775 rtx rhs_rtx = expand_normal (rhs); 2778 rtx rhs_rtx = expand_normal (rhs);
2776 2779
2777 struct expand_operand ops[6]; 2780 class expand_operand ops[6];
2778 int i = 0; 2781 int i = 0;
2779 create_address_operand (&ops[i++], base_rtx); 2782 create_address_operand (&ops[i++], base_rtx);
2780 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); 2783 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
2781 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); 2784 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
2782 create_integer_operand (&ops[i++], scale_int); 2785 create_integer_operand (&ops[i++], scale_int);
2786 tree mask = gimple_call_arg (stmt, mask_index); 2789 tree mask = gimple_call_arg (stmt, mask_index);
2787 rtx mask_rtx = expand_normal (mask); 2790 rtx mask_rtx = expand_normal (mask);
2788 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); 2791 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
2789 } 2792 }
2790 2793
2791 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs))); 2794 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
2795 TYPE_MODE (TREE_TYPE (offset)));
2792 expand_insn (icode, i, ops); 2796 expand_insn (icode, i, ops);
2793 } 2797 }
2794 2798
2795 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */ 2799 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
2796 2800
2806 rtx base_rtx = expand_normal (base); 2810 rtx base_rtx = expand_normal (base);
2807 rtx offset_rtx = expand_normal (offset); 2811 rtx offset_rtx = expand_normal (offset);
2808 HOST_WIDE_INT scale_int = tree_to_shwi (scale); 2812 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
2809 2813
2810 int i = 0; 2814 int i = 0;
2811 struct expand_operand ops[6]; 2815 class expand_operand ops[6];
2812 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs))); 2816 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
2813 create_address_operand (&ops[i++], base_rtx); 2817 create_address_operand (&ops[i++], base_rtx);
2814 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); 2818 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
2815 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); 2819 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
2816 create_integer_operand (&ops[i++], scale_int); 2820 create_integer_operand (&ops[i++], scale_int);
2817 if (optab == mask_gather_load_optab) 2821 if (optab == mask_gather_load_optab)
2818 { 2822 {
2819 tree mask = gimple_call_arg (stmt, 3); 2823 tree mask = gimple_call_arg (stmt, 4);
2820 rtx mask_rtx = expand_normal (mask); 2824 rtx mask_rtx = expand_normal (mask);
2821 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); 2825 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
2822 } 2826 }
2823 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))); 2827 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
2828 TYPE_MODE (TREE_TYPE (offset)));
2824 expand_insn (icode, i, ops); 2829 expand_insn (icode, i, ops);
2825 } 2830 }
2826 2831
2827 /* Expand DIVMOD() using: 2832 /* Expand DIVMOD() using:
2828 a) optab handler for udivmod/sdivmod if it is available. 2833 a) optab handler for udivmod/sdivmod if it is available.
2875 2880
2876 static void 2881 static void
2877 expand_NOP (internal_fn, gcall *) 2882 expand_NOP (internal_fn, gcall *)
2878 { 2883 {
2879 /* Nothing. But it shouldn't really prevail. */ 2884 /* Nothing. But it shouldn't really prevail. */
2885 }
2886
2887 /* Coroutines, all should have been processed at this stage. */
2888
2889 static void
2890 expand_CO_FRAME (internal_fn, gcall *)
2891 {
2892 gcc_unreachable ();
2893 }
2894
2895 static void
2896 expand_CO_YIELD (internal_fn, gcall *)
2897 {
2898 gcc_unreachable ();
2899 }
2900
2901 static void
2902 expand_CO_SUSPN (internal_fn, gcall *)
2903 {
2904 gcc_unreachable ();
2905 }
2906
2907 static void
2908 expand_CO_ACTOR (internal_fn, gcall *)
2909 {
2910 gcc_unreachable ();
2880 } 2911 }
2881 2912
2882 /* Expand a call to FN using the operands in STMT. FN has a single 2913 /* Expand a call to FN using the operands in STMT. FN has a single
2883 output operand and NARGS input operands. */ 2914 output operand and NARGS input operands. */
2884 2915
3002 expand_direct_optab_fn (FN, STMT, OPTAB, 3) 3033 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3003 3034
3004 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \ 3035 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3005 expand_direct_optab_fn (FN, STMT, OPTAB, 2) 3036 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3006 3037
3038 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3039 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3040
3041 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3042 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3043
3007 /* RETURN_TYPE and ARGS are a return type and argument list that are 3044 /* RETURN_TYPE and ARGS are a return type and argument list that are
3008 in principle compatible with FN (which satisfies direct_internal_fn_p). 3045 in principle compatible with FN (which satisfies direct_internal_fn_p).
3009 Return the types that should be used to determine whether the 3046 Return the types that should be used to determine whether the
3010 target supports FN. */ 3047 target supports FN. */
3011 3048
3082 #define direct_cond_binary_optab_supported_p direct_optab_supported_p 3119 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3083 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p 3120 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3084 #define direct_mask_load_optab_supported_p direct_optab_supported_p 3121 #define direct_mask_load_optab_supported_p direct_optab_supported_p
3085 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p 3122 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3086 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p 3123 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3087 #define direct_gather_load_optab_supported_p direct_optab_supported_p 3124 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3088 #define direct_mask_store_optab_supported_p direct_optab_supported_p 3125 #define direct_mask_store_optab_supported_p direct_optab_supported_p
3089 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p 3126 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3090 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p 3127 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3091 #define direct_scatter_store_optab_supported_p direct_optab_supported_p 3128 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3092 #define direct_while_optab_supported_p convert_optab_supported_p 3129 #define direct_while_optab_supported_p convert_optab_supported_p
3093 #define direct_fold_extract_optab_supported_p direct_optab_supported_p 3130 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3094 #define direct_fold_left_optab_supported_p direct_optab_supported_p 3131 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3132 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3133 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3095 3134
3096 /* Return the optab used by internal function FN. */ 3135 /* Return the optab used by internal function FN. */
3097 3136
3098 static optab 3137 static optab
3099 direct_internal_fn_optab (internal_fn fn, tree_pair types) 3138 direct_internal_fn_optab (internal_fn fn, tree_pair types)
3207 case IFN_FMS: 3246 case IFN_FMS:
3208 case IFN_FNMA: 3247 case IFN_FNMA:
3209 case IFN_FNMS: 3248 case IFN_FNMS:
3210 case IFN_AVG_FLOOR: 3249 case IFN_AVG_FLOOR:
3211 case IFN_AVG_CEIL: 3250 case IFN_AVG_CEIL:
3251 case IFN_MULHS:
3252 case IFN_MULHRS:
3212 case IFN_FMIN: 3253 case IFN_FMIN:
3213 case IFN_FMAX: 3254 case IFN_FMAX:
3214 return 0; 3255 return 0;
3215 3256
3216 case IFN_COND_ADD: 3257 case IFN_COND_ADD:
3283 T (RDIV_EXPR, IFN_COND_RDIV) \ 3324 T (RDIV_EXPR, IFN_COND_RDIV) \
3284 T (MIN_EXPR, IFN_COND_MIN) \ 3325 T (MIN_EXPR, IFN_COND_MIN) \
3285 T (MAX_EXPR, IFN_COND_MAX) \ 3326 T (MAX_EXPR, IFN_COND_MAX) \
3286 T (BIT_AND_EXPR, IFN_COND_AND) \ 3327 T (BIT_AND_EXPR, IFN_COND_AND) \
3287 T (BIT_IOR_EXPR, IFN_COND_IOR) \ 3328 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3288 T (BIT_XOR_EXPR, IFN_COND_XOR) 3329 T (BIT_XOR_EXPR, IFN_COND_XOR) \
3330 T (LSHIFT_EXPR, IFN_COND_SHL) \
3331 T (RSHIFT_EXPR, IFN_COND_SHR)
3289 3332
3290 /* Return a function that only performs CODE when a certain condition is met 3333 /* Return a function that only performs CODE when a certain condition is met
3291 and that uses a given fallback value otherwise. For example, if CODE is 3334 and that uses a given fallback value otherwise. For example, if CODE is
3292 a binary operation associated with conditional function FN: 3335 a binary operation associated with conditional function FN:
3293 3336
3506 case IFN_MASK_STORE: 3549 case IFN_MASK_STORE:
3507 case IFN_MASK_STORE_LANES: 3550 case IFN_MASK_STORE_LANES:
3508 return 2; 3551 return 2;
3509 3552
3510 case IFN_MASK_GATHER_LOAD: 3553 case IFN_MASK_GATHER_LOAD:
3511 return 3;
3512
3513 case IFN_MASK_SCATTER_STORE: 3554 case IFN_MASK_SCATTER_STORE:
3514 return 4; 3555 return 4;
3515 3556
3516 default: 3557 default:
3517 return (conditional_internal_fn_code (fn) != ERROR_MARK 3558 return (conditional_internal_fn_code (fn) != ERROR_MARK
3539 3580
3540 /* Return true if the target supports gather load or scatter store function 3581 /* Return true if the target supports gather load or scatter store function
3541 IFN. For loads, VECTOR_TYPE is the vector type of the load result, 3582 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
3542 while for stores it is the vector type of the stored data argument. 3583 while for stores it is the vector type of the stored data argument.
3543 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded 3584 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
3544 or stored. OFFSET_SIGN is the sign of the offset argument, which is 3585 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
3545 only relevant when the offset is narrower than an address. SCALE is 3586 offset from the shared base address of each loaded or stored element.
3546 the amount by which the offset should be multiplied *after* it has 3587 SCALE is the amount by which these offsets should be multiplied
3547 been extended to address width. */ 3588 *after* they have been extended to address width. */
3548 3589
3549 bool 3590 bool
3550 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type, 3591 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
3551 tree memory_element_type, 3592 tree memory_element_type,
3552 signop offset_sign, int scale) 3593 tree offset_vector_type, int scale)
3553 { 3594 {
3554 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)), 3595 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
3555 TYPE_SIZE (memory_element_type))) 3596 TYPE_SIZE (memory_element_type)))
3556 return false; 3597 return false;
3598 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
3599 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
3600 return false;
3557 optab optab = direct_internal_fn_optab (ifn); 3601 optab optab = direct_internal_fn_optab (ifn);
3558 insn_code icode = direct_optab_handler (optab, TYPE_MODE (vector_type)); 3602 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
3603 TYPE_MODE (offset_vector_type));
3559 int output_ops = internal_load_fn_p (ifn) ? 1 : 0; 3604 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
3605 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
3560 return (icode != CODE_FOR_nothing 3606 return (icode != CODE_FOR_nothing
3561 && insn_operand_matches (icode, 2 + output_ops, 3607 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
3562 GEN_INT (offset_sign == UNSIGNED)) 3608 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
3563 && insn_operand_matches (icode, 3 + output_ops, 3609 }
3564 GEN_INT (scale))); 3610
3611 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
3612 for pointers of type TYPE when the accesses have LENGTH bytes and their
3613 common byte alignment is ALIGN. */
3614
3615 bool
3616 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
3617 poly_uint64 length, unsigned int align)
3618 {
3619 machine_mode mode = TYPE_MODE (type);
3620 optab optab = direct_internal_fn_optab (ifn);
3621 insn_code icode = direct_optab_handler (optab, mode);
3622 if (icode == CODE_FOR_nothing)
3623 return false;
3624 rtx length_rtx = immed_wide_int_const (length, mode);
3625 return (insn_operand_matches (icode, 3, length_rtx)
3626 && insn_operand_matches (icode, 4, GEN_INT (align)));
3565 } 3627 }
3566 3628
3567 /* Expand STMT as though it were a call to internal function FN. */ 3629 /* Expand STMT as though it were a call to internal function FN. */
3568 3630
3569 void 3631 void