Mercurial > hg > CbC > CbC_gcc
comparison gcc/internal-fn.c @ 145:1830386684a0
gcc-9.2.0
author | anatofuz |
---|---|
date | Thu, 13 Feb 2020 11:34:05 +0900 |
parents | 84e7813d76e9 |
children |
comparison
equal
deleted
inserted
replaced
131:84e7813d76e9 | 145:1830386684a0 |
---|---|
1 /* Internal functions. | 1 /* Internal functions. |
2 Copyright (C) 2011-2018 Free Software Foundation, Inc. | 2 Copyright (C) 2011-2020 Free Software Foundation, Inc. |
3 | 3 |
4 This file is part of GCC. | 4 This file is part of GCC. |
5 | 5 |
6 GCC is free software; you can redistribute it and/or modify it under | 6 GCC is free software; you can redistribute it and/or modify it under |
7 the terms of the GNU General Public License as published by the Free | 7 the terms of the GNU General Public License as published by the Free |
101 direct_internal_fn. */ | 101 direct_internal_fn. */ |
102 #define not_direct { -2, -2, false } | 102 #define not_direct { -2, -2, false } |
103 #define mask_load_direct { -1, 2, false } | 103 #define mask_load_direct { -1, 2, false } |
104 #define load_lanes_direct { -1, -1, false } | 104 #define load_lanes_direct { -1, -1, false } |
105 #define mask_load_lanes_direct { -1, -1, false } | 105 #define mask_load_lanes_direct { -1, -1, false } |
106 #define gather_load_direct { -1, -1, false } | 106 #define gather_load_direct { 3, 1, false } |
107 #define mask_store_direct { 3, 2, false } | 107 #define mask_store_direct { 3, 2, false } |
108 #define store_lanes_direct { 0, 0, false } | 108 #define store_lanes_direct { 0, 0, false } |
109 #define mask_store_lanes_direct { 0, 0, false } | 109 #define mask_store_lanes_direct { 0, 0, false } |
110 #define scatter_store_direct { 3, 3, false } | 110 #define scatter_store_direct { 3, 1, false } |
111 #define unary_direct { 0, 0, true } | 111 #define unary_direct { 0, 0, true } |
112 #define binary_direct { 0, 0, true } | 112 #define binary_direct { 0, 0, true } |
113 #define ternary_direct { 0, 0, true } | 113 #define ternary_direct { 0, 0, true } |
114 #define cond_unary_direct { 1, 1, true } | 114 #define cond_unary_direct { 1, 1, true } |
115 #define cond_binary_direct { 1, 1, true } | 115 #define cond_binary_direct { 1, 1, true } |
116 #define cond_ternary_direct { 1, 1, true } | 116 #define cond_ternary_direct { 1, 1, true } |
117 #define while_direct { 0, 2, false } | 117 #define while_direct { 0, 2, false } |
118 #define fold_extract_direct { 2, 2, false } | 118 #define fold_extract_direct { 2, 2, false } |
119 #define fold_left_direct { 1, 1, false } | 119 #define fold_left_direct { 1, 1, false } |
120 #define mask_fold_left_direct { 1, 1, false } | |
121 #define check_ptrs_direct { 0, 0, false } | |
120 | 122 |
121 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = { | 123 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = { |
122 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct, | 124 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct, |
123 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct, | 125 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct, |
124 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \ | 126 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \ |
146 /* Expand LOAD_LANES call STMT using optab OPTAB. */ | 148 /* Expand LOAD_LANES call STMT using optab OPTAB. */ |
147 | 149 |
148 static void | 150 static void |
149 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) | 151 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) |
150 { | 152 { |
151 struct expand_operand ops[2]; | 153 class expand_operand ops[2]; |
152 tree type, lhs, rhs; | 154 tree type, lhs, rhs; |
153 rtx target, mem; | 155 rtx target, mem; |
154 | 156 |
155 lhs = gimple_call_lhs (stmt); | 157 lhs = gimple_call_lhs (stmt); |
156 rhs = gimple_call_arg (stmt, 0); | 158 rhs = gimple_call_arg (stmt, 0); |
170 /* Expand STORE_LANES call STMT using optab OPTAB. */ | 172 /* Expand STORE_LANES call STMT using optab OPTAB. */ |
171 | 173 |
172 static void | 174 static void |
173 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) | 175 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab) |
174 { | 176 { |
175 struct expand_operand ops[2]; | 177 class expand_operand ops[2]; |
176 tree type, lhs, rhs; | 178 tree type, lhs, rhs; |
177 rtx target, reg; | 179 rtx target, reg; |
178 | 180 |
179 lhs = gimple_call_lhs (stmt); | 181 lhs = gimple_call_lhs (stmt); |
180 rhs = gimple_call_arg (stmt, 0); | 182 rhs = gimple_call_arg (stmt, 0); |
224 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 226 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
225 else | 227 else |
226 target = gen_reg_rtx (Pmode); | 228 target = gen_reg_rtx (Pmode); |
227 rtx size = expand_normal (gimple_call_arg (stmt, 0)); | 229 rtx size = expand_normal (gimple_call_arg (stmt, 0)); |
228 rtx align = expand_normal (gimple_call_arg (stmt, 1)); | 230 rtx align = expand_normal (gimple_call_arg (stmt, 1)); |
229 struct expand_operand ops[3]; | 231 class expand_operand ops[3]; |
230 create_output_operand (&ops[0], target, Pmode); | 232 create_output_operand (&ops[0], target, Pmode); |
231 create_input_operand (&ops[1], size, Pmode); | 233 create_input_operand (&ops[1], size, Pmode); |
232 create_input_operand (&ops[2], align, Pmode); | 234 create_input_operand (&ops[2], align, Pmode); |
233 gcc_assert (targetm.have_omp_simt_enter ()); | 235 gcc_assert (targetm.have_omp_simt_enter ()); |
234 expand_insn (targetm.code_for_omp_simt_enter, 3, ops); | 236 expand_insn (targetm.code_for_omp_simt_enter, 3, ops); |
239 static void | 241 static void |
240 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt) | 242 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt) |
241 { | 243 { |
242 gcc_checking_assert (!gimple_call_lhs (stmt)); | 244 gcc_checking_assert (!gimple_call_lhs (stmt)); |
243 rtx arg = expand_normal (gimple_call_arg (stmt, 0)); | 245 rtx arg = expand_normal (gimple_call_arg (stmt, 0)); |
244 struct expand_operand ops[1]; | 246 class expand_operand ops[1]; |
245 create_input_operand (&ops[0], arg, Pmode); | 247 create_input_operand (&ops[0], arg, Pmode); |
246 gcc_assert (targetm.have_omp_simt_exit ()); | 248 gcc_assert (targetm.have_omp_simt_exit ()); |
247 expand_insn (targetm.code_for_omp_simt_exit, 1, ops); | 249 expand_insn (targetm.code_for_omp_simt_exit, 1, ops); |
248 } | 250 } |
249 | 251 |
282 return; | 284 return; |
283 | 285 |
284 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 286 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
285 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); | 287 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); |
286 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | 288 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
287 struct expand_operand ops[2]; | 289 class expand_operand ops[2]; |
288 create_output_operand (&ops[0], target, mode); | 290 create_output_operand (&ops[0], target, mode); |
289 create_input_operand (&ops[1], cond, mode); | 291 create_input_operand (&ops[1], cond, mode); |
290 gcc_assert (targetm.have_omp_simt_last_lane ()); | 292 gcc_assert (targetm.have_omp_simt_last_lane ()); |
291 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops); | 293 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops); |
292 } | 294 } |
301 return; | 303 return; |
302 | 304 |
303 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 305 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
304 rtx ctr = expand_normal (gimple_call_arg (stmt, 0)); | 306 rtx ctr = expand_normal (gimple_call_arg (stmt, 0)); |
305 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | 307 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
306 struct expand_operand ops[2]; | 308 class expand_operand ops[2]; |
307 create_output_operand (&ops[0], target, mode); | 309 create_output_operand (&ops[0], target, mode); |
308 create_input_operand (&ops[1], ctr, mode); | 310 create_input_operand (&ops[1], ctr, mode); |
309 gcc_assert (targetm.have_omp_simt_ordered ()); | 311 gcc_assert (targetm.have_omp_simt_ordered ()); |
310 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops); | 312 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops); |
311 } | 313 } |
321 return; | 323 return; |
322 | 324 |
323 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 325 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
324 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); | 326 rtx cond = expand_normal (gimple_call_arg (stmt, 0)); |
325 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | 327 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
326 struct expand_operand ops[2]; | 328 class expand_operand ops[2]; |
327 create_output_operand (&ops[0], target, mode); | 329 create_output_operand (&ops[0], target, mode); |
328 create_input_operand (&ops[1], cond, mode); | 330 create_input_operand (&ops[1], cond, mode); |
329 gcc_assert (targetm.have_omp_simt_vote_any ()); | 331 gcc_assert (targetm.have_omp_simt_vote_any ()); |
330 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops); | 332 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops); |
331 } | 333 } |
342 | 344 |
343 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 345 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
344 rtx src = expand_normal (gimple_call_arg (stmt, 0)); | 346 rtx src = expand_normal (gimple_call_arg (stmt, 0)); |
345 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); | 347 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); |
346 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | 348 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
347 struct expand_operand ops[3]; | 349 class expand_operand ops[3]; |
348 create_output_operand (&ops[0], target, mode); | 350 create_output_operand (&ops[0], target, mode); |
349 create_input_operand (&ops[1], src, mode); | 351 create_input_operand (&ops[1], src, mode); |
350 create_input_operand (&ops[2], idx, SImode); | 352 create_input_operand (&ops[2], idx, SImode); |
351 gcc_assert (targetm.have_omp_simt_xchg_bfly ()); | 353 gcc_assert (targetm.have_omp_simt_xchg_bfly ()); |
352 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops); | 354 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops); |
363 | 365 |
364 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); | 366 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE); |
365 rtx src = expand_normal (gimple_call_arg (stmt, 0)); | 367 rtx src = expand_normal (gimple_call_arg (stmt, 0)); |
366 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); | 368 rtx idx = expand_normal (gimple_call_arg (stmt, 1)); |
367 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | 369 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
368 struct expand_operand ops[3]; | 370 class expand_operand ops[3]; |
369 create_output_operand (&ops[0], target, mode); | 371 create_output_operand (&ops[0], target, mode); |
370 create_input_operand (&ops[1], src, mode); | 372 create_input_operand (&ops[1], src, mode); |
371 create_input_operand (&ops[2], idx, SImode); | 373 create_input_operand (&ops[2], idx, SImode); |
372 gcc_assert (targetm.have_omp_simt_xchg_idx ()); | 374 gcc_assert (targetm.have_omp_simt_xchg_idx ()); |
373 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops); | 375 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops); |
771 { | 773 { |
772 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab | 774 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab |
773 : usubv4_optab, mode); | 775 : usubv4_optab, mode); |
774 if (icode != CODE_FOR_nothing) | 776 if (icode != CODE_FOR_nothing) |
775 { | 777 { |
776 struct expand_operand ops[4]; | 778 class expand_operand ops[4]; |
777 rtx_insn *last = get_last_insn (); | 779 rtx_insn *last = get_last_insn (); |
778 | 780 |
779 res = gen_reg_rtx (mode); | 781 res = gen_reg_rtx (mode); |
780 create_output_operand (&ops[0], res, mode); | 782 create_output_operand (&ops[0], res, mode); |
781 create_input_operand (&ops[1], op0, mode); | 783 create_input_operand (&ops[1], op0, mode); |
992 { | 994 { |
993 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab | 995 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab |
994 : subv4_optab, mode); | 996 : subv4_optab, mode); |
995 if (icode != CODE_FOR_nothing) | 997 if (icode != CODE_FOR_nothing) |
996 { | 998 { |
997 struct expand_operand ops[4]; | 999 class expand_operand ops[4]; |
998 rtx_insn *last = get_last_insn (); | 1000 rtx_insn *last = get_last_insn (); |
999 | 1001 |
1000 res = gen_reg_rtx (mode); | 1002 res = gen_reg_rtx (mode); |
1001 create_output_operand (&ops[0], res, mode); | 1003 create_output_operand (&ops[0], res, mode); |
1002 create_input_operand (&ops[1], op0, mode); | 1004 create_input_operand (&ops[1], op0, mode); |
1143 } | 1145 } |
1144 | 1146 |
1145 enum insn_code icode = optab_handler (negv3_optab, mode); | 1147 enum insn_code icode = optab_handler (negv3_optab, mode); |
1146 if (icode != CODE_FOR_nothing) | 1148 if (icode != CODE_FOR_nothing) |
1147 { | 1149 { |
1148 struct expand_operand ops[3]; | 1150 class expand_operand ops[3]; |
1149 rtx_insn *last = get_last_insn (); | 1151 rtx_insn *last = get_last_insn (); |
1150 | 1152 |
1151 res = gen_reg_rtx (mode); | 1153 res = gen_reg_rtx (mode); |
1152 create_output_operand (&ops[0], res, mode); | 1154 create_output_operand (&ops[0], res, mode); |
1153 create_input_operand (&ops[1], op1, mode); | 1155 create_input_operand (&ops[1], op1, mode); |
1405 } | 1407 } |
1406 | 1408 |
1407 /* s1 * s2 -> ur */ | 1409 /* s1 * s2 -> ur */ |
1408 if (!uns0_p && !uns1_p && unsr_p) | 1410 if (!uns0_p && !uns1_p && unsr_p) |
1409 { | 1411 { |
1410 rtx tem, tem2; | 1412 rtx tem; |
1411 switch (pos_neg0 | pos_neg1) | 1413 switch (pos_neg0 | pos_neg1) |
1412 { | 1414 { |
1413 case 1: /* Both operands known to be non-negative. */ | 1415 case 1: /* Both operands known to be non-negative. */ |
1414 goto do_main; | 1416 goto do_main; |
1415 case 2: /* Both operands known to be negative. */ | 1417 case 2: /* Both operands known to be negative. */ |
1435 ops.op0 = make_tree (ops.type, op0); | 1437 ops.op0 = make_tree (ops.type, op0); |
1436 ops.op1 = make_tree (ops.type, op1); | 1438 ops.op1 = make_tree (ops.type, op1); |
1437 ops.op2 = NULL_TREE; | 1439 ops.op2 = NULL_TREE; |
1438 ops.location = loc; | 1440 ops.location = loc; |
1439 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | 1441 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); |
1440 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false, | 1442 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ, |
1441 OPTAB_LIB_WIDEN); | 1443 true, mode, NULL_RTX, NULL, done_label, |
1442 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, | |
1443 NULL_RTX, NULL, done_label, | |
1444 profile_probability::very_likely ()); | 1444 profile_probability::very_likely ()); |
1445 goto do_error_label; | 1445 goto do_error_label; |
1446 } | 1446 } |
1447 /* The general case, do all the needed comparisons at runtime. */ | 1447 /* The general case, do all the needed comparisons at runtime. */ |
1448 rtx_code_label *do_main_label, *after_negate_label; | 1448 rtx_code_label *do_main_label, *after_negate_label; |
1469 the arguments. */ | 1469 the arguments. */ |
1470 arg0 = error_mark_node; | 1470 arg0 = error_mark_node; |
1471 arg1 = error_mark_node; | 1471 arg1 = error_mark_node; |
1472 emit_jump (do_main_label); | 1472 emit_jump (do_main_label); |
1473 emit_label (after_negate_label); | 1473 emit_label (after_negate_label); |
1474 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false, | 1474 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false, |
1475 OPTAB_LIB_WIDEN); | 1475 OPTAB_LIB_WIDEN); |
1476 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX, | 1476 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX, |
1477 NULL, do_main_label, profile_probability::very_likely ()); | 1477 NULL, do_main_label, |
1478 profile_probability::very_likely ()); | |
1478 /* One argument is negative here, the other positive. This | 1479 /* One argument is negative here, the other positive. This |
1479 overflows always, unless one of the arguments is 0. But | 1480 overflows always, unless one of the arguments is 0. But |
1480 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1 | 1481 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1 |
1481 is, thus we can keep do_main code oring in overflow as is. */ | 1482 is, thus we can keep do_main code oring in overflow as is. */ |
1482 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX, | 1483 if (pos_neg0 != 2) |
1483 NULL, do_main_label, profile_probability::very_likely ()); | 1484 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX, |
1485 NULL, do_main_label, | |
1486 profile_probability::very_unlikely ()); | |
1487 if (pos_neg1 != 2) | |
1488 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX, | |
1489 NULL, do_main_label, | |
1490 profile_probability::very_unlikely ()); | |
1484 expand_arith_set_overflow (lhs, target); | 1491 expand_arith_set_overflow (lhs, target); |
1485 emit_label (do_main_label); | 1492 emit_label (do_main_label); |
1486 goto do_main; | 1493 goto do_main; |
1487 default: | 1494 default: |
1488 gcc_unreachable (); | 1495 gcc_unreachable (); |
1536 goto do_error_label; | 1543 goto do_error_label; |
1537 } | 1544 } |
1538 } | 1545 } |
1539 if (icode != CODE_FOR_nothing) | 1546 if (icode != CODE_FOR_nothing) |
1540 { | 1547 { |
1541 struct expand_operand ops[4]; | 1548 class expand_operand ops[4]; |
1542 rtx_insn *last = get_last_insn (); | 1549 rtx_insn *last = get_last_insn (); |
1543 | 1550 |
1544 res = gen_reg_rtx (mode); | 1551 res = gen_reg_rtx (mode); |
1545 create_output_operand (&ops[0], res, mode); | 1552 create_output_operand (&ops[0], res, mode); |
1546 create_input_operand (&ops[1], op0, mode); | 1553 create_input_operand (&ops[1], op0, mode); |
1751 profile_probability::unlikely ()); | 1758 profile_probability::unlikely ()); |
1752 | 1759 |
1753 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from | 1760 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from |
1754 hmode to mode, the multiplication will never overflow. We can | 1761 hmode to mode, the multiplication will never overflow. We can |
1755 do just one hmode x hmode => mode widening multiplication. */ | 1762 do just one hmode x hmode => mode widening multiplication. */ |
1756 rtx lopart0s = lopart0, lopart1s = lopart1; | |
1757 if (GET_CODE (lopart0) == SUBREG) | |
1758 { | |
1759 lopart0s = shallow_copy_rtx (lopart0); | |
1760 SUBREG_PROMOTED_VAR_P (lopart0s) = 1; | |
1761 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED); | |
1762 } | |
1763 if (GET_CODE (lopart1) == SUBREG) | |
1764 { | |
1765 lopart1s = shallow_copy_rtx (lopart1); | |
1766 SUBREG_PROMOTED_VAR_P (lopart1s) = 1; | |
1767 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED); | |
1768 } | |
1769 tree halfstype = build_nonstandard_integer_type (hprec, uns); | 1763 tree halfstype = build_nonstandard_integer_type (hprec, uns); |
1770 ops.op0 = make_tree (halfstype, lopart0s); | 1764 ops.op0 = make_tree (halfstype, lopart0); |
1771 ops.op1 = make_tree (halfstype, lopart1s); | 1765 ops.op1 = make_tree (halfstype, lopart1); |
1772 ops.code = WIDEN_MULT_EXPR; | 1766 ops.code = WIDEN_MULT_EXPR; |
1773 ops.type = type; | 1767 ops.type = type; |
1774 rtx thisres | 1768 rtx thisres |
1775 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); | 1769 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL); |
1776 emit_move_insn (res, thisres); | 1770 emit_move_insn (res, thisres); |
2485 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */ | 2479 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */ |
2486 | 2480 |
2487 static void | 2481 static void |
2488 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab) | 2482 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab) |
2489 { | 2483 { |
2490 struct expand_operand ops[3]; | 2484 class expand_operand ops[3]; |
2491 tree type, lhs, rhs, maskt; | 2485 tree type, lhs, rhs, maskt; |
2492 rtx mem, target, mask; | 2486 rtx mem, target, mask; |
2493 insn_code icode; | 2487 insn_code icode; |
2494 | 2488 |
2495 maskt = gimple_call_arg (stmt, 2); | 2489 maskt = gimple_call_arg (stmt, 2); |
2520 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */ | 2514 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */ |
2521 | 2515 |
2522 static void | 2516 static void |
2523 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab) | 2517 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab) |
2524 { | 2518 { |
2525 struct expand_operand ops[3]; | 2519 class expand_operand ops[3]; |
2526 tree type, lhs, rhs, maskt; | 2520 tree type, lhs, rhs, maskt; |
2527 rtx mem, reg, mask; | 2521 rtx mem, reg, mask; |
2528 insn_code icode; | 2522 insn_code icode; |
2529 | 2523 |
2530 maskt = gimple_call_arg (stmt, 2); | 2524 maskt = gimple_call_arg (stmt, 2); |
2575 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function | 2569 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function |
2576 should never be called. */ | 2570 should never be called. */ |
2577 | 2571 |
2578 static void | 2572 static void |
2579 expand_VA_ARG (internal_fn, gcall *) | 2573 expand_VA_ARG (internal_fn, gcall *) |
2574 { | |
2575 gcc_unreachable (); | |
2576 } | |
2577 | |
2578 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this | |
2579 dummy function should never be called. */ | |
2580 | |
2581 static void | |
2582 expand_VEC_CONVERT (internal_fn, gcall *) | |
2580 { | 2583 { |
2581 gcc_unreachable (); | 2584 gcc_unreachable (); |
2582 } | 2585 } |
2583 | 2586 |
2584 /* Expand the IFN_UNIQUE function according to its first argument. */ | 2587 /* Expand the IFN_UNIQUE function according to its first argument. */ |
2772 rtx base_rtx = expand_normal (base); | 2775 rtx base_rtx = expand_normal (base); |
2773 rtx offset_rtx = expand_normal (offset); | 2776 rtx offset_rtx = expand_normal (offset); |
2774 HOST_WIDE_INT scale_int = tree_to_shwi (scale); | 2777 HOST_WIDE_INT scale_int = tree_to_shwi (scale); |
2775 rtx rhs_rtx = expand_normal (rhs); | 2778 rtx rhs_rtx = expand_normal (rhs); |
2776 | 2779 |
2777 struct expand_operand ops[6]; | 2780 class expand_operand ops[6]; |
2778 int i = 0; | 2781 int i = 0; |
2779 create_address_operand (&ops[i++], base_rtx); | 2782 create_address_operand (&ops[i++], base_rtx); |
2780 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); | 2783 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); |
2781 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); | 2784 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); |
2782 create_integer_operand (&ops[i++], scale_int); | 2785 create_integer_operand (&ops[i++], scale_int); |
2786 tree mask = gimple_call_arg (stmt, mask_index); | 2789 tree mask = gimple_call_arg (stmt, mask_index); |
2787 rtx mask_rtx = expand_normal (mask); | 2790 rtx mask_rtx = expand_normal (mask); |
2788 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); | 2791 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); |
2789 } | 2792 } |
2790 | 2793 |
2791 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs))); | 2794 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)), |
2795 TYPE_MODE (TREE_TYPE (offset))); | |
2792 expand_insn (icode, i, ops); | 2796 expand_insn (icode, i, ops); |
2793 } | 2797 } |
2794 | 2798 |
2795 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */ | 2799 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */ |
2796 | 2800 |
2806 rtx base_rtx = expand_normal (base); | 2810 rtx base_rtx = expand_normal (base); |
2807 rtx offset_rtx = expand_normal (offset); | 2811 rtx offset_rtx = expand_normal (offset); |
2808 HOST_WIDE_INT scale_int = tree_to_shwi (scale); | 2812 HOST_WIDE_INT scale_int = tree_to_shwi (scale); |
2809 | 2813 |
2810 int i = 0; | 2814 int i = 0; |
2811 struct expand_operand ops[6]; | 2815 class expand_operand ops[6]; |
2812 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs))); | 2816 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs))); |
2813 create_address_operand (&ops[i++], base_rtx); | 2817 create_address_operand (&ops[i++], base_rtx); |
2814 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); | 2818 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset))); |
2815 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); | 2819 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset))); |
2816 create_integer_operand (&ops[i++], scale_int); | 2820 create_integer_operand (&ops[i++], scale_int); |
2817 if (optab == mask_gather_load_optab) | 2821 if (optab == mask_gather_load_optab) |
2818 { | 2822 { |
2819 tree mask = gimple_call_arg (stmt, 3); | 2823 tree mask = gimple_call_arg (stmt, 4); |
2820 rtx mask_rtx = expand_normal (mask); | 2824 rtx mask_rtx = expand_normal (mask); |
2821 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); | 2825 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask))); |
2822 } | 2826 } |
2823 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))); | 2827 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)), |
2828 TYPE_MODE (TREE_TYPE (offset))); | |
2824 expand_insn (icode, i, ops); | 2829 expand_insn (icode, i, ops); |
2825 } | 2830 } |
2826 | 2831 |
2827 /* Expand DIVMOD() using: | 2832 /* Expand DIVMOD() using: |
2828 a) optab handler for udivmod/sdivmod if it is available. | 2833 a) optab handler for udivmod/sdivmod if it is available. |
2875 | 2880 |
2876 static void | 2881 static void |
2877 expand_NOP (internal_fn, gcall *) | 2882 expand_NOP (internal_fn, gcall *) |
2878 { | 2883 { |
2879 /* Nothing. But it shouldn't really prevail. */ | 2884 /* Nothing. But it shouldn't really prevail. */ |
2885 } | |
2886 | |
2887 /* Coroutines, all should have been processed at this stage. */ | |
2888 | |
2889 static void | |
2890 expand_CO_FRAME (internal_fn, gcall *) | |
2891 { | |
2892 gcc_unreachable (); | |
2893 } | |
2894 | |
2895 static void | |
2896 expand_CO_YIELD (internal_fn, gcall *) | |
2897 { | |
2898 gcc_unreachable (); | |
2899 } | |
2900 | |
2901 static void | |
2902 expand_CO_SUSPN (internal_fn, gcall *) | |
2903 { | |
2904 gcc_unreachable (); | |
2905 } | |
2906 | |
2907 static void | |
2908 expand_CO_ACTOR (internal_fn, gcall *) | |
2909 { | |
2910 gcc_unreachable (); | |
2880 } | 2911 } |
2881 | 2912 |
2882 /* Expand a call to FN using the operands in STMT. FN has a single | 2913 /* Expand a call to FN using the operands in STMT. FN has a single |
2883 output operand and NARGS input operands. */ | 2914 output operand and NARGS input operands. */ |
2884 | 2915 |
3002 expand_direct_optab_fn (FN, STMT, OPTAB, 3) | 3033 expand_direct_optab_fn (FN, STMT, OPTAB, 3) |
3003 | 3034 |
3004 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \ | 3035 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \ |
3005 expand_direct_optab_fn (FN, STMT, OPTAB, 2) | 3036 expand_direct_optab_fn (FN, STMT, OPTAB, 2) |
3006 | 3037 |
3038 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \ | |
3039 expand_direct_optab_fn (FN, STMT, OPTAB, 3) | |
3040 | |
3041 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \ | |
3042 expand_direct_optab_fn (FN, STMT, OPTAB, 4) | |
3043 | |
3007 /* RETURN_TYPE and ARGS are a return type and argument list that are | 3044 /* RETURN_TYPE and ARGS are a return type and argument list that are |
3008 in principle compatible with FN (which satisfies direct_internal_fn_p). | 3045 in principle compatible with FN (which satisfies direct_internal_fn_p). |
3009 Return the types that should be used to determine whether the | 3046 Return the types that should be used to determine whether the |
3010 target supports FN. */ | 3047 target supports FN. */ |
3011 | 3048 |
3082 #define direct_cond_binary_optab_supported_p direct_optab_supported_p | 3119 #define direct_cond_binary_optab_supported_p direct_optab_supported_p |
3083 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p | 3120 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p |
3084 #define direct_mask_load_optab_supported_p direct_optab_supported_p | 3121 #define direct_mask_load_optab_supported_p direct_optab_supported_p |
3085 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p | 3122 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p |
3086 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p | 3123 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p |
3087 #define direct_gather_load_optab_supported_p direct_optab_supported_p | 3124 #define direct_gather_load_optab_supported_p convert_optab_supported_p |
3088 #define direct_mask_store_optab_supported_p direct_optab_supported_p | 3125 #define direct_mask_store_optab_supported_p direct_optab_supported_p |
3089 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p | 3126 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p |
3090 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p | 3127 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p |
3091 #define direct_scatter_store_optab_supported_p direct_optab_supported_p | 3128 #define direct_scatter_store_optab_supported_p convert_optab_supported_p |
3092 #define direct_while_optab_supported_p convert_optab_supported_p | 3129 #define direct_while_optab_supported_p convert_optab_supported_p |
3093 #define direct_fold_extract_optab_supported_p direct_optab_supported_p | 3130 #define direct_fold_extract_optab_supported_p direct_optab_supported_p |
3094 #define direct_fold_left_optab_supported_p direct_optab_supported_p | 3131 #define direct_fold_left_optab_supported_p direct_optab_supported_p |
3132 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p | |
3133 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p | |
3095 | 3134 |
3096 /* Return the optab used by internal function FN. */ | 3135 /* Return the optab used by internal function FN. */ |
3097 | 3136 |
3098 static optab | 3137 static optab |
3099 direct_internal_fn_optab (internal_fn fn, tree_pair types) | 3138 direct_internal_fn_optab (internal_fn fn, tree_pair types) |
3207 case IFN_FMS: | 3246 case IFN_FMS: |
3208 case IFN_FNMA: | 3247 case IFN_FNMA: |
3209 case IFN_FNMS: | 3248 case IFN_FNMS: |
3210 case IFN_AVG_FLOOR: | 3249 case IFN_AVG_FLOOR: |
3211 case IFN_AVG_CEIL: | 3250 case IFN_AVG_CEIL: |
3251 case IFN_MULHS: | |
3252 case IFN_MULHRS: | |
3212 case IFN_FMIN: | 3253 case IFN_FMIN: |
3213 case IFN_FMAX: | 3254 case IFN_FMAX: |
3214 return 0; | 3255 return 0; |
3215 | 3256 |
3216 case IFN_COND_ADD: | 3257 case IFN_COND_ADD: |
3283 T (RDIV_EXPR, IFN_COND_RDIV) \ | 3324 T (RDIV_EXPR, IFN_COND_RDIV) \ |
3284 T (MIN_EXPR, IFN_COND_MIN) \ | 3325 T (MIN_EXPR, IFN_COND_MIN) \ |
3285 T (MAX_EXPR, IFN_COND_MAX) \ | 3326 T (MAX_EXPR, IFN_COND_MAX) \ |
3286 T (BIT_AND_EXPR, IFN_COND_AND) \ | 3327 T (BIT_AND_EXPR, IFN_COND_AND) \ |
3287 T (BIT_IOR_EXPR, IFN_COND_IOR) \ | 3328 T (BIT_IOR_EXPR, IFN_COND_IOR) \ |
3288 T (BIT_XOR_EXPR, IFN_COND_XOR) | 3329 T (BIT_XOR_EXPR, IFN_COND_XOR) \ |
3330 T (LSHIFT_EXPR, IFN_COND_SHL) \ | |
3331 T (RSHIFT_EXPR, IFN_COND_SHR) | |
3289 | 3332 |
3290 /* Return a function that only performs CODE when a certain condition is met | 3333 /* Return a function that only performs CODE when a certain condition is met |
3291 and that uses a given fallback value otherwise. For example, if CODE is | 3334 and that uses a given fallback value otherwise. For example, if CODE is |
3292 a binary operation associated with conditional function FN: | 3335 a binary operation associated with conditional function FN: |
3293 | 3336 |
3506 case IFN_MASK_STORE: | 3549 case IFN_MASK_STORE: |
3507 case IFN_MASK_STORE_LANES: | 3550 case IFN_MASK_STORE_LANES: |
3508 return 2; | 3551 return 2; |
3509 | 3552 |
3510 case IFN_MASK_GATHER_LOAD: | 3553 case IFN_MASK_GATHER_LOAD: |
3511 return 3; | |
3512 | |
3513 case IFN_MASK_SCATTER_STORE: | 3554 case IFN_MASK_SCATTER_STORE: |
3514 return 4; | 3555 return 4; |
3515 | 3556 |
3516 default: | 3557 default: |
3517 return (conditional_internal_fn_code (fn) != ERROR_MARK | 3558 return (conditional_internal_fn_code (fn) != ERROR_MARK |
3539 | 3580 |
3540 /* Return true if the target supports gather load or scatter store function | 3581 /* Return true if the target supports gather load or scatter store function |
3541 IFN. For loads, VECTOR_TYPE is the vector type of the load result, | 3582 IFN. For loads, VECTOR_TYPE is the vector type of the load result, |
3542 while for stores it is the vector type of the stored data argument. | 3583 while for stores it is the vector type of the stored data argument. |
3543 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded | 3584 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded |
3544 or stored. OFFSET_SIGN is the sign of the offset argument, which is | 3585 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the |
3545 only relevant when the offset is narrower than an address. SCALE is | 3586 offset from the shared base address of each loaded or stored element. |
3546 the amount by which the offset should be multiplied *after* it has | 3587 SCALE is the amount by which these offsets should be multiplied |
3547 been extended to address width. */ | 3588 *after* they have been extended to address width. */ |
3548 | 3589 |
3549 bool | 3590 bool |
3550 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type, | 3591 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type, |
3551 tree memory_element_type, | 3592 tree memory_element_type, |
3552 signop offset_sign, int scale) | 3593 tree offset_vector_type, int scale) |
3553 { | 3594 { |
3554 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)), | 3595 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)), |
3555 TYPE_SIZE (memory_element_type))) | 3596 TYPE_SIZE (memory_element_type))) |
3556 return false; | 3597 return false; |
3598 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type), | |
3599 TYPE_VECTOR_SUBPARTS (offset_vector_type))) | |
3600 return false; | |
3557 optab optab = direct_internal_fn_optab (ifn); | 3601 optab optab = direct_internal_fn_optab (ifn); |
3558 insn_code icode = direct_optab_handler (optab, TYPE_MODE (vector_type)); | 3602 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type), |
3603 TYPE_MODE (offset_vector_type)); | |
3559 int output_ops = internal_load_fn_p (ifn) ? 1 : 0; | 3604 int output_ops = internal_load_fn_p (ifn) ? 1 : 0; |
3605 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type)); | |
3560 return (icode != CODE_FOR_nothing | 3606 return (icode != CODE_FOR_nothing |
3561 && insn_operand_matches (icode, 2 + output_ops, | 3607 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p)) |
3562 GEN_INT (offset_sign == UNSIGNED)) | 3608 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale))); |
3563 && insn_operand_matches (icode, 3 + output_ops, | 3609 } |
3564 GEN_INT (scale))); | 3610 |
3611 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN | |
3612 for pointers of type TYPE when the accesses have LENGTH bytes and their | |
3613 common byte alignment is ALIGN. */ | |
3614 | |
3615 bool | |
3616 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type, | |
3617 poly_uint64 length, unsigned int align) | |
3618 { | |
3619 machine_mode mode = TYPE_MODE (type); | |
3620 optab optab = direct_internal_fn_optab (ifn); | |
3621 insn_code icode = direct_optab_handler (optab, mode); | |
3622 if (icode == CODE_FOR_nothing) | |
3623 return false; | |
3624 rtx length_rtx = immed_wide_int_const (length, mode); | |
3625 return (insn_operand_matches (icode, 3, length_rtx) | |
3626 && insn_operand_matches (icode, 4, GEN_INT (align))); | |
3565 } | 3627 } |
3566 | 3628 |
3567 /* Expand STMT as though it were a call to internal function FN. */ | 3629 /* Expand STMT as though it were a call to internal function FN. */ |
3568 | 3630 |
3569 void | 3631 void |