Mercurial > hg > CbC > CbC_gcc
diff gcc/optabs.c @ 111:04ced10e8804
gcc 7
| author | kono |
|---|---|
| date | Fri, 27 Oct 2017 22:46:09 +0900 |
| parents | f6334be47118 |
| children | 84e7813d76e9 |
line wrap: on
line diff
--- a/gcc/optabs.c Sun Aug 21 07:07:55 2011 +0900 +++ b/gcc/optabs.c Fri Oct 27 22:46:09 2017 +0900 @@ -1,7 +1,5 @@ /* Expand the basic unary and binary arithmetic operations, for GNU compiler. - Copyright (C) 1987, 1988, 1992, 1993, 1994, 1995, 1996, 1997, 1998, - 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 - Free Software Foundation, Inc. + Copyright (C) 1987-2017 Free Software Foundation, Inc. This file is part of GCC. @@ -23,139 +21,36 @@ #include "config.h" #include "system.h" #include "coretypes.h" -#include "tm.h" +#include "backend.h" +#include "target.h" +#include "rtl.h" +#include "tree.h" +#include "memmodel.h" +#include "predict.h" +#include "tm_p.h" +#include "expmed.h" +#include "optabs.h" +#include "emit-rtl.h" +#include "recog.h" #include "diagnostic-core.h" /* Include insn-config.h before expr.h so that HAVE_conditional_move is properly defined. */ -#include "insn-config.h" -#include "rtl.h" -#include "tree.h" -#include "tm_p.h" -#include "flags.h" -#include "function.h" +#include "stor-layout.h" #include "except.h" +#include "dojump.h" +#include "explow.h" #include "expr.h" -#include "optabs.h" +#include "optabs-tree.h" #include "libfuncs.h" -#include "recog.h" -#include "reload.h" -#include "ggc.h" -#include "basic-block.h" -#include "target.h" - -struct target_optabs default_target_optabs; -struct target_libfuncs default_target_libfuncs; -#if SWITCHABLE_TARGET -struct target_optabs *this_target_optabs = &default_target_optabs; -struct target_libfuncs *this_target_libfuncs = &default_target_libfuncs; -#endif - -#define libfunc_hash \ - (this_target_libfuncs->x_libfunc_hash) - -/* Contains the optab used for each rtx code. */ -optab code_to_optab[NUM_RTX_CODE + 1]; static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *, - enum machine_mode *); -static rtx expand_unop_direct (enum machine_mode, optab, rtx, rtx, int); + machine_mode *); +static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int); +static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool); /* Debug facility for use in GDB. */ void debug_optab_libfuncs (void); - -/* Prefixes for the current version of decimal floating point (BID vs. DPD) */ -#if ENABLE_DECIMAL_BID_FORMAT -#define DECIMAL_PREFIX "bid_" -#else -#define DECIMAL_PREFIX "dpd_" -#endif - -/* Used for libfunc_hash. */ - -static hashval_t -hash_libfunc (const void *p) -{ - const struct libfunc_entry *const e = (const struct libfunc_entry *) p; - - return (((int) e->mode1 + (int) e->mode2 * NUM_MACHINE_MODES) - ^ e->optab); -} - -/* Used for libfunc_hash. */ - -static int -eq_libfunc (const void *p, const void *q) -{ - const struct libfunc_entry *const e1 = (const struct libfunc_entry *) p; - const struct libfunc_entry *const e2 = (const struct libfunc_entry *) q; - - return (e1->optab == e2->optab - && e1->mode1 == e2->mode1 - && e1->mode2 == e2->mode2); -} - -/* Return libfunc corresponding operation defined by OPTAB converting - from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL - if no libfunc is available. */ -rtx -convert_optab_libfunc (convert_optab optab, enum machine_mode mode1, - enum machine_mode mode2) -{ - struct libfunc_entry e; - struct libfunc_entry **slot; - - e.optab = (size_t) (optab - &convert_optab_table[0]); - e.mode1 = mode1; - e.mode2 = mode2; - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, NO_INSERT); - if (!slot) - { - if (optab->libcall_gen) - { - optab->libcall_gen (optab, optab->libcall_basename, mode1, mode2); - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, NO_INSERT); - if (slot) - return (*slot)->libfunc; - else - return NULL; - } - return NULL; - } - return (*slot)->libfunc; -} - -/* Return libfunc corresponding operation defined by OPTAB in MODE. - Trigger lazy initialization if needed, return NULL if no libfunc is - available. */ -rtx -optab_libfunc (optab optab, enum machine_mode mode) -{ - struct libfunc_entry e; - struct libfunc_entry **slot; - - e.optab = (size_t) (optab - &optab_table[0]); - e.mode1 = mode; - e.mode2 = VOIDmode; - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, NO_INSERT); - if (!slot) - { - if (optab->libcall_gen) - { - optab->libcall_gen (optab, optab->libcall_basename, - optab->libcall_suffix, mode); - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, - &e, NO_INSERT); - if (slot) - return (*slot)->libfunc; - else - return NULL; - } - return NULL; - } - return (*slot)->libfunc; -} - /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to the result of operation CODE applied to OP0 (and OP1 if it is a binary @@ -163,14 +58,15 @@ If the last insn does not set TARGET, don't do anything, but return 1. - If a previous insn sets TARGET and TARGET is one of OP0 or OP1, - don't add the REG_EQUAL note but return 0. Our caller can then try - again, ensuring that TARGET is not one of the operands. */ + If the last insn or a previous insn sets TARGET and TARGET is one of OP0 + or OP1, don't add the REG_EQUAL note but return 0. Our caller can then + try again, ensuring that TARGET is not one of the operands. */ static int -add_equal_note (rtx insns, rtx target, enum rtx_code code, rtx op0, rtx op1) +add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, rtx op1) { - rtx last_insn, insn, set; + rtx_insn *last_insn; + rtx set; rtx note; gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns)); @@ -190,7 +86,36 @@ last_insn = NEXT_INSN (last_insn)) ; - set = single_set (last_insn); + /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing + a value changing in the insn, so the note would be invalid for CSE. */ + if (reg_overlap_mentioned_p (target, op0) + || (op1 && reg_overlap_mentioned_p (target, op1))) + { + if (MEM_P (target) + && (rtx_equal_p (target, op0) + || (op1 && rtx_equal_p (target, op1)))) + { + /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note + over expanding it as temp = MEM op X, MEM = temp. If the target + supports MEM = MEM op X instructions, it is sometimes too hard + to reconstruct that form later, especially if X is also a memory, + and due to multiple occurrences of addresses the address might + be forced into register unnecessarily. + Note that not emitting the REG_EQUIV note might inhibit + CSE in some cases. */ + set = single_set (last_insn); + if (set + && GET_CODE (SET_SRC (set)) == code + && MEM_P (SET_DEST (set)) + && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0)) + || (op1 && rtx_equal_p (SET_DEST (set), + XEXP (SET_SRC (set), 1))))) + return 1; + } + return 0; + } + + set = set_for_reg_notes (last_insn); if (set == NULL_RTX) return 1; @@ -200,23 +125,33 @@ || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target))) return 1; - /* If TARGET is in OP0 or OP1, check if anything in SEQ sets TARGET - besides the last insn. */ - if (reg_overlap_mentioned_p (target, op0) - || (op1 && reg_overlap_mentioned_p (target, op1))) - { - insn = PREV_INSN (last_insn); - while (insn != NULL_RTX) - { - if (reg_set_p (target, insn)) - return 0; - - insn = PREV_INSN (insn); - } - } - if (GET_RTX_CLASS (code) == RTX_UNARY) - note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0)); + switch (code) + { + case FFS: + case CLZ: + case CTZ: + case CLRSB: + case POPCOUNT: + case PARITY: + case BSWAP: + if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0)) + { + note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0)); + if (GET_MODE_UNIT_SIZE (GET_MODE (op0)) + > GET_MODE_UNIT_SIZE (GET_MODE (target))) + note = simplify_gen_unary (TRUNCATE, GET_MODE (target), + note, GET_MODE (op0)); + else + note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target), + note, GET_MODE (op0)); + break; + } + /* FALLTHRU */ + default: + note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0)); + break; + } else note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1)); @@ -225,6 +160,30 @@ return 1; } +/* Given two input operands, OP0 and OP1, determine what the correct from_mode + for a widening operation would be. In most cases this would be OP0, but if + that's a constant it'll be VOIDmode, which isn't useful. */ + +static machine_mode +widened_mode (machine_mode to_mode, rtx op0, rtx op1) +{ + machine_mode m0 = GET_MODE (op0); + machine_mode m1 = GET_MODE (op1); + machine_mode result; + + if (m0 == VOIDmode && m1 == VOIDmode) + return to_mode; + else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1)) + result = m1; + else + result = m0; + + if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode)) + return to_mode; + + return result; +} + /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need not actually do a sign-extend or zero-extend, but can leave the @@ -232,10 +191,11 @@ of logical operations, but not right shifts. */ static rtx -widen_operand (rtx op, enum machine_mode mode, enum machine_mode oldmode, +widen_operand (rtx op, machine_mode mode, machine_mode oldmode, int unsignedp, int no_extend) { rtx result; + scalar_int_mode int_mode; /* If we don't have to extend and this is a constant, return it. */ if (no_extend && GET_MODE (op) == VOIDmode) @@ -245,238 +205,25 @@ extend since it will be more efficient to do so unless the signedness of a promoted object differs from our extension. */ if (! no_extend + || !is_a <scalar_int_mode> (mode, &int_mode) || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op) - && SUBREG_PROMOTED_UNSIGNED_P (op) == unsignedp)) + && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp))) return convert_modes (mode, oldmode, op, unsignedp); - /* If MODE is no wider than a single word, we return a paradoxical + /* If MODE is no wider than a single word, we return a lowpart or paradoxical SUBREG. */ - if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) - return gen_rtx_SUBREG (mode, force_reg (GET_MODE (op), op), 0); + if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD) + return gen_lowpart (int_mode, force_reg (GET_MODE (op), op)); /* Otherwise, get an object of MODE, clobber it, and set the low-order part to OP. */ - result = gen_reg_rtx (mode); + result = gen_reg_rtx (int_mode); emit_clobber (result); emit_move_insn (gen_lowpart (GET_MODE (op), result), op); return result; } -/* Return the optab used for computing the operation given by the tree code, - CODE and the tree EXP. This function is not always usable (for example, it - cannot give complete results for multiplication or division) but probably - ought to be relied on more widely throughout the expander. */ -optab -optab_for_tree_code (enum tree_code code, const_tree type, - enum optab_subtype subtype) -{ - bool trapv; - switch (code) - { - case BIT_AND_EXPR: - return and_optab; - - case BIT_IOR_EXPR: - return ior_optab; - - case BIT_NOT_EXPR: - return one_cmpl_optab; - - case BIT_XOR_EXPR: - return xor_optab; - - case TRUNC_MOD_EXPR: - case CEIL_MOD_EXPR: - case FLOOR_MOD_EXPR: - case ROUND_MOD_EXPR: - return TYPE_UNSIGNED (type) ? umod_optab : smod_optab; - - case RDIV_EXPR: - case TRUNC_DIV_EXPR: - case CEIL_DIV_EXPR: - case FLOOR_DIV_EXPR: - case ROUND_DIV_EXPR: - case EXACT_DIV_EXPR: - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? usdiv_optab : ssdiv_optab; - return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab; - - case LSHIFT_EXPR: - if (TREE_CODE (type) == VECTOR_TYPE) - { - if (subtype == optab_vector) - return TYPE_SATURATING (type) ? NULL : vashl_optab; - - gcc_assert (subtype == optab_scalar); - } - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? usashl_optab : ssashl_optab; - return ashl_optab; - - case RSHIFT_EXPR: - if (TREE_CODE (type) == VECTOR_TYPE) - { - if (subtype == optab_vector) - return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab; - - gcc_assert (subtype == optab_scalar); - } - return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab; - - case LROTATE_EXPR: - if (TREE_CODE (type) == VECTOR_TYPE) - { - if (subtype == optab_vector) - return vrotl_optab; - - gcc_assert (subtype == optab_scalar); - } - return rotl_optab; - - case RROTATE_EXPR: - if (TREE_CODE (type) == VECTOR_TYPE) - { - if (subtype == optab_vector) - return vrotr_optab; - - gcc_assert (subtype == optab_scalar); - } - return rotr_optab; - - case MAX_EXPR: - return TYPE_UNSIGNED (type) ? umax_optab : smax_optab; - - case MIN_EXPR: - return TYPE_UNSIGNED (type) ? umin_optab : smin_optab; - - case REALIGN_LOAD_EXPR: - return vec_realign_load_optab; - - case WIDEN_SUM_EXPR: - return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab; - - case DOT_PROD_EXPR: - return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab; - - case WIDEN_MULT_PLUS_EXPR: - return (TYPE_UNSIGNED (type) - ? (TYPE_SATURATING (type) - ? usmadd_widen_optab : umadd_widen_optab) - : (TYPE_SATURATING (type) - ? ssmadd_widen_optab : smadd_widen_optab)); - - case WIDEN_MULT_MINUS_EXPR: - return (TYPE_UNSIGNED (type) - ? (TYPE_SATURATING (type) - ? usmsub_widen_optab : umsub_widen_optab) - : (TYPE_SATURATING (type) - ? ssmsub_widen_optab : smsub_widen_optab)); - - case FMA_EXPR: - return fma_optab; - - case REDUC_MAX_EXPR: - return TYPE_UNSIGNED (type) ? reduc_umax_optab : reduc_smax_optab; - - case REDUC_MIN_EXPR: - return TYPE_UNSIGNED (type) ? reduc_umin_optab : reduc_smin_optab; - - case REDUC_PLUS_EXPR: - return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab; - - case VEC_LSHIFT_EXPR: - return vec_shl_optab; - - case VEC_RSHIFT_EXPR: - return vec_shr_optab; - - case VEC_WIDEN_MULT_HI_EXPR: - return TYPE_UNSIGNED (type) ? - vec_widen_umult_hi_optab : vec_widen_smult_hi_optab; - - case VEC_WIDEN_MULT_LO_EXPR: - return TYPE_UNSIGNED (type) ? - vec_widen_umult_lo_optab : vec_widen_smult_lo_optab; - - case VEC_UNPACK_HI_EXPR: - return TYPE_UNSIGNED (type) ? - vec_unpacku_hi_optab : vec_unpacks_hi_optab; - - case VEC_UNPACK_LO_EXPR: - return TYPE_UNSIGNED (type) ? - vec_unpacku_lo_optab : vec_unpacks_lo_optab; - - case VEC_UNPACK_FLOAT_HI_EXPR: - /* The signedness is determined from input operand. */ - return TYPE_UNSIGNED (type) ? - vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab; - - case VEC_UNPACK_FLOAT_LO_EXPR: - /* The signedness is determined from input operand. */ - return TYPE_UNSIGNED (type) ? - vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab; - - case VEC_PACK_TRUNC_EXPR: - return vec_pack_trunc_optab; - - case VEC_PACK_SAT_EXPR: - return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab; - - case VEC_PACK_FIX_TRUNC_EXPR: - /* The signedness is determined from output operand. */ - return TYPE_UNSIGNED (type) ? - vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab; - - default: - break; - } - - trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type); - switch (code) - { - case POINTER_PLUS_EXPR: - case PLUS_EXPR: - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? usadd_optab : ssadd_optab; - return trapv ? addv_optab : add_optab; - - case MINUS_EXPR: - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? ussub_optab : sssub_optab; - return trapv ? subv_optab : sub_optab; - - case MULT_EXPR: - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? usmul_optab : ssmul_optab; - return trapv ? smulv_optab : smul_optab; - - case NEGATE_EXPR: - if (TYPE_SATURATING(type)) - return TYPE_UNSIGNED(type) ? usneg_optab : ssneg_optab; - return trapv ? negv_optab : neg_optab; - - case ABS_EXPR: - return trapv ? absv_optab : abs_optab; - - case VEC_EXTRACT_EVEN_EXPR: - return vec_extract_even_optab; - - case VEC_EXTRACT_ODD_EXPR: - return vec_extract_odd_optab; - - case VEC_INTERLEAVE_HIGH_EXPR: - return vec_interleave_high_optab; - - case VEC_INTERLEAVE_LOW_EXPR: - return vec_interleave_low_optab; - - default: - return NULL; - } -} - - /* Expand vector widening operations. There are two different classes of operations handled here: @@ -501,15 +248,13 @@ expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, rtx target, int unsignedp) { + struct expand_operand eops[4]; tree oprnd0, oprnd1, oprnd2; - enum machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode; + machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode; optab widen_pattern_optab; - int icode; - enum machine_mode xmode0, xmode1 = VOIDmode, wxmode = VOIDmode; - rtx temp; - rtx pat; - rtx xop0, xop1, wxop; + enum insn_code icode; int nops = TREE_CODE_LENGTH (ops->code); + int op; oprnd0 = ops->op0; tmode0 = TYPE_MODE (TREE_TYPE (oprnd0)); @@ -517,117 +262,39 @@ optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default); if (ops->code == WIDEN_MULT_PLUS_EXPR || ops->code == WIDEN_MULT_MINUS_EXPR) - icode = (int) optab_handler (widen_pattern_optab, - TYPE_MODE (TREE_TYPE (ops->op2))); + icode = find_widening_optab_handler (widen_pattern_optab, + TYPE_MODE (TREE_TYPE (ops->op2)), + tmode0, 0); else - icode = (int) optab_handler (widen_pattern_optab, tmode0); + icode = optab_handler (widen_pattern_optab, tmode0); gcc_assert (icode != CODE_FOR_nothing); - xmode0 = insn_data[icode].operand[1].mode; if (nops >= 2) { oprnd1 = ops->op1; tmode1 = TYPE_MODE (TREE_TYPE (oprnd1)); - xmode1 = insn_data[icode].operand[2].mode; } /* The last operand is of a wider mode than the rest of the operands. */ if (nops == 2) - { - wmode = tmode1; - wxmode = xmode1; - } + wmode = tmode1; else if (nops == 3) { gcc_assert (tmode1 == tmode0); gcc_assert (op1); oprnd2 = ops->op2; wmode = TYPE_MODE (TREE_TYPE (oprnd2)); - wxmode = insn_data[icode].operand[3].mode; - } - - if (!wide_op) - wmode = wxmode = insn_data[icode].operand[0].mode; - - if (!target - || ! (*insn_data[icode].operand[0].predicate) (target, wmode)) - temp = gen_reg_rtx (wmode); - else - temp = target; - - xop0 = op0; - xop1 = op1; - wxop = wide_op; - - /* In case the insn wants input operands in modes different from - those of the actual operands, convert the operands. It would - seem that we don't need to convert CONST_INTs, but we do, so - that they're properly zero-extended, sign-extended or truncated - for their mode. */ - - if (GET_MODE (op0) != xmode0 && xmode0 != VOIDmode) - xop0 = convert_modes (xmode0, - GET_MODE (op0) != VOIDmode - ? GET_MODE (op0) - : tmode0, - xop0, unsignedp); - + } + + op = 0; + create_output_operand (&eops[op++], target, TYPE_MODE (ops->type)); + create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp); if (op1) - if (GET_MODE (op1) != xmode1 && xmode1 != VOIDmode) - xop1 = convert_modes (xmode1, - GET_MODE (op1) != VOIDmode - ? GET_MODE (op1) - : tmode1, - xop1, unsignedp); - + create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp); if (wide_op) - if (GET_MODE (wide_op) != wxmode && wxmode != VOIDmode) - wxop = convert_modes (wxmode, - GET_MODE (wide_op) != VOIDmode - ? GET_MODE (wide_op) - : wmode, - wxop, unsignedp); - - /* Now, if insn's predicates don't allow our operands, put them into - pseudo regs. */ - - if (! (*insn_data[icode].operand[1].predicate) (xop0, xmode0) - && xmode0 != VOIDmode) - xop0 = copy_to_mode_reg (xmode0, xop0); - - if (op1) - { - if (! (*insn_data[icode].operand[2].predicate) (xop1, xmode1) - && xmode1 != VOIDmode) - xop1 = copy_to_mode_reg (xmode1, xop1); - - if (wide_op) - { - if (! (*insn_data[icode].operand[3].predicate) (wxop, wxmode) - && wxmode != VOIDmode) - wxop = copy_to_mode_reg (wxmode, wxop); - - pat = GEN_FCN (icode) (temp, xop0, xop1, wxop); - } - else - pat = GEN_FCN (icode) (temp, xop0, xop1); - } - else - { - if (wide_op) - { - if (! (*insn_data[icode].operand[2].predicate) (wxop, wxmode) - && wxmode != VOIDmode) - wxop = copy_to_mode_reg (wxmode, wxop); - - pat = GEN_FCN (icode) (temp, xop0, wxop); - } - else - pat = GEN_FCN (icode) (temp, xop0); - } - - emit_insn (pat); - return temp; + create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp); + expand_insn (icode, op, eops); + return eops[0].value; } /* Generate code to perform an operation specified by TERNARY_OPTAB @@ -642,70 +309,20 @@ this may or may not be TARGET. */ rtx -expand_ternary_op (enum machine_mode mode, optab ternary_optab, rtx op0, +expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0, rtx op1, rtx op2, rtx target, int unsignedp) { - int icode = (int) optab_handler (ternary_optab, mode); - enum machine_mode mode0 = insn_data[icode].operand[1].mode; - enum machine_mode mode1 = insn_data[icode].operand[2].mode; - enum machine_mode mode2 = insn_data[icode].operand[3].mode; - rtx temp; - rtx pat; - rtx xop0 = op0, xop1 = op1, xop2 = op2; + struct expand_operand ops[4]; + enum insn_code icode = optab_handler (ternary_optab, mode); gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing); - if (!target || !insn_data[icode].operand[0].predicate (target, mode)) - temp = gen_reg_rtx (mode); - else - temp = target; - - /* In case the insn wants input operands in modes different from - those of the actual operands, convert the operands. It would - seem that we don't need to convert CONST_INTs, but we do, so - that they're properly zero-extended, sign-extended or truncated - for their mode. */ - - if (GET_MODE (op0) != mode0 && mode0 != VOIDmode) - xop0 = convert_modes (mode0, - GET_MODE (op0) != VOIDmode - ? GET_MODE (op0) - : mode, - xop0, unsignedp); - - if (GET_MODE (op1) != mode1 && mode1 != VOIDmode) - xop1 = convert_modes (mode1, - GET_MODE (op1) != VOIDmode - ? GET_MODE (op1) - : mode, - xop1, unsignedp); - - if (GET_MODE (op2) != mode2 && mode2 != VOIDmode) - xop2 = convert_modes (mode2, - GET_MODE (op2) != VOIDmode - ? GET_MODE (op2) - : mode, - xop2, unsignedp); - - /* Now, if insn's predicates don't allow our operands, put them into - pseudo regs. */ - - if (!insn_data[icode].operand[1].predicate (xop0, mode0) - && mode0 != VOIDmode) - xop0 = copy_to_mode_reg (mode0, xop0); - - if (!insn_data[icode].operand[2].predicate (xop1, mode1) - && mode1 != VOIDmode) - xop1 = copy_to_mode_reg (mode1, xop1); - - if (!insn_data[icode].operand[3].predicate (xop2, mode2) - && mode2 != VOIDmode) - xop2 = copy_to_mode_reg (mode2, xop2); - - pat = GEN_FCN (icode) (temp, xop0, xop1, xop2); - - emit_insn (pat); - return temp; + create_output_operand (&ops[0], target, mode); + create_convert_operand_from (&ops[1], op0, mode, unsignedp); + create_convert_operand_from (&ops[2], op1, mode, unsignedp); + create_convert_operand_from (&ops[3], op2, mode, unsignedp); + expand_insn (icode, 4, ops); + return ops[0].value; } @@ -713,15 +330,15 @@ calculated at compile time. The arguments and return value are otherwise the same as for expand_binop. */ -static rtx -simplify_expand_binop (enum machine_mode mode, optab binoptab, +rtx +simplify_expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods) { if (CONSTANT_P (op0) && CONSTANT_P (op1)) { - rtx x = simplify_binary_operation (binoptab->code, mode, op0, op1); - + rtx x = simplify_binary_operation (optab_to_code (binoptab), + mode, op0, op1); if (x) return x; } @@ -733,7 +350,7 @@ Return true if the expansion succeeded. */ bool -force_expand_binop (enum machine_mode mode, optab binoptab, +force_expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods) { @@ -746,59 +363,40 @@ return true; } -/* Generate insns for VEC_LSHIFT_EXPR, VEC_RSHIFT_EXPR. */ - -rtx -expand_vec_shift_expr (sepops ops, rtx target) +/* Create a new vector value in VMODE with all elements set to OP. The + mode of OP must be the element mode of VMODE. If OP is a constant, + then the return value will be a constant. */ + +static rtx +expand_vector_broadcast (machine_mode vmode, rtx op) { enum insn_code icode; - rtx rtx_op1, rtx_op2; - enum machine_mode mode1; - enum machine_mode mode2; - enum machine_mode mode = TYPE_MODE (ops->type); - tree vec_oprnd = ops->op0; - tree shift_oprnd = ops->op1; - optab shift_optab; - rtx pat; - - switch (ops->code) - { - case VEC_RSHIFT_EXPR: - shift_optab = vec_shr_optab; - break; - case VEC_LSHIFT_EXPR: - shift_optab = vec_shl_optab; - break; - default: - gcc_unreachable (); - } - - icode = optab_handler (shift_optab, mode); - gcc_assert (icode != CODE_FOR_nothing); - - mode1 = insn_data[icode].operand[1].mode; - mode2 = insn_data[icode].operand[2].mode; - - rtx_op1 = expand_normal (vec_oprnd); - if (!(*insn_data[icode].operand[1].predicate) (rtx_op1, mode1) - && mode1 != VOIDmode) - rtx_op1 = force_reg (mode1, rtx_op1); - - rtx_op2 = expand_normal (shift_oprnd); - if (!(*insn_data[icode].operand[2].predicate) (rtx_op2, mode2) - && mode2 != VOIDmode) - rtx_op2 = force_reg (mode2, rtx_op2); - - if (!target - || ! (*insn_data[icode].operand[0].predicate) (target, mode)) - target = gen_reg_rtx (mode); - - /* Emit instruction */ - pat = GEN_FCN (icode) (target, rtx_op1, rtx_op2); - gcc_assert (pat); - emit_insn (pat); - - return target; + rtvec vec; + rtx ret; + int i, n; + + gcc_checking_assert (VECTOR_MODE_P (vmode)); + + n = GET_MODE_NUNITS (vmode); + vec = rtvec_alloc (n); + for (i = 0; i < n; ++i) + RTVEC_ELT (vec, i) = op; + + if (CONSTANT_P (op)) + return gen_rtx_CONST_VECTOR (vmode, vec); + + /* ??? If the target doesn't have a vec_init, then we have no easy way + of performing this operation. Most of this sort of generic support + is hidden away in the vector lowering support in gimple. */ + icode = convert_optab_handler (vec_init_optab, vmode, + GET_MODE_INNER (vmode)); + if (icode == CODE_FOR_nothing) + return NULL; + + ret = gen_reg_rtx (vmode); + emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec))); + + return ret; } /* This subroutine of expand_doubleword_shift handles the cases in which @@ -837,7 +435,7 @@ value are the same as for the parent routine. */ static bool -expand_subword_shift (enum machine_mode op1_mode, optab binoptab, +expand_subword_shift (scalar_int_mode op1_mode, optab binoptab, rtx outof_input, rtx into_input, rtx op1, rtx outof_target, rtx into_target, int unsignedp, enum optab_methods methods, @@ -855,7 +453,8 @@ if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD) { carries = outof_input; - tmp = immed_double_const (BITS_PER_WORD, 0, op1_mode); + tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, + op1_mode), op1_mode); tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, 0, true, methods); } @@ -870,13 +469,15 @@ outof_input, const1_rtx, 0, unsignedp, methods); if (shift_mask == BITS_PER_WORD - 1) { - tmp = immed_double_const (-1, -1, op1_mode); + tmp = immed_wide_int_const + (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode); tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp, 0, true, methods); } else { - tmp = immed_double_const (BITS_PER_WORD - 1, 0, op1_mode); + tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1, + op1_mode), op1_mode); tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, 0, true, methods); } @@ -910,7 +511,6 @@ } -#ifdef HAVE_conditional_move /* Try implementing expand_doubleword_shift using conditional moves. The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true, otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1 @@ -918,7 +518,7 @@ arguments are the same as the parent routine. */ static bool -expand_doubleword_shift_condmove (enum machine_mode op1_mode, optab binoptab, +expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab, enum rtx_code cmp_code, rtx cmp1, rtx cmp2, rtx outof_input, rtx into_input, rtx subword_op1, rtx superword_op1, @@ -970,7 +570,6 @@ return true; } -#endif /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts. OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first @@ -1002,14 +601,13 @@ Return true if the shift could be successfully synthesized. */ static bool -expand_doubleword_shift (enum machine_mode op1_mode, optab binoptab, +expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab, rtx outof_input, rtx into_input, rtx op1, rtx outof_target, rtx into_target, int unsignedp, enum optab_methods methods, unsigned HOST_WIDE_INT shift_mask) { rtx superword_op1, tmp, cmp1, cmp2; - rtx subword_label, done_label; enum rtx_code cmp_code; /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will @@ -1039,7 +637,7 @@ is true when the effective shift value is less than BITS_PER_WORD. Set SUPERWORD_OP1 to the shift count that should be used to shift OUTOF_INPUT into INTO_TARGET when the condition is false. */ - tmp = immed_double_const (BITS_PER_WORD, 0, op1_mode); + tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode); if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1) { /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1 @@ -1078,28 +676,28 @@ unsignedp, methods, shift_mask); } -#ifdef HAVE_conditional_move /* Try using conditional moves to generate straight-line code. */ - { - rtx start = get_last_insn (); - if (expand_doubleword_shift_condmove (op1_mode, binoptab, - cmp_code, cmp1, cmp2, - outof_input, into_input, - op1, superword_op1, - outof_target, into_target, - unsignedp, methods, shift_mask)) - return true; - delete_insns_since (start); - } -#endif + if (HAVE_conditional_move) + { + rtx_insn *start = get_last_insn (); + if (expand_doubleword_shift_condmove (op1_mode, binoptab, + cmp_code, cmp1, cmp2, + outof_input, into_input, + op1, superword_op1, + outof_target, into_target, + unsignedp, methods, shift_mask)) + return true; + delete_insns_since (start); + } /* As a last resort, use branches to select the correct alternative. */ - subword_label = gen_label_rtx (); - done_label = gen_label_rtx (); + rtx_code_label *subword_label = gen_label_rtx (); + rtx_code_label *done_label = gen_label_rtx (); NO_DEFER_POP; do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode, - 0, 0, subword_label, -1); + 0, 0, subword_label, + profile_probability::uninitialized ()); OK_DEFER_POP; if (!expand_superword_shift (binoptab, outof_input, superword_op1, @@ -1107,7 +705,7 @@ unsignedp, methods)) return false; - emit_jump_insn (gen_jump (done_label)); + emit_jump_insn (targetm.gen_jump (done_label)); emit_barrier (); emit_label (subword_label); @@ -1177,7 +775,7 @@ the 0 or -1. */ static rtx -expand_doubleword_mult (enum machine_mode mode, rtx op0, rtx op1, rtx target, +expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target, bool umulp, enum optab_methods methods) { int low = (WORDS_BIG_ENDIAN ? 1 : 0); @@ -1285,11 +883,11 @@ the operation to perform, not an optab pointer. All other arguments are the same. */ rtx -expand_simple_binop (enum machine_mode mode, enum rtx_code code, rtx op0, +expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods) { - optab binop = code_to_optab[(int) code]; + optab binop = code_to_optab (code); gcc_assert (binop); return expand_binop (mode, binop, op0, op1, target, unsignedp, methods); @@ -1323,7 +921,7 @@ static bool shift_optab_p (optab binoptab) { - switch (binoptab->code) + switch (optab_to_code (binoptab)) { case ASHIFT: case SS_ASHIFT: @@ -1344,28 +942,29 @@ static bool commutative_optab_p (optab binoptab) { - return (GET_RTX_CLASS (binoptab->code) == RTX_COMM_ARITH + return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH || binoptab == smul_widen_optab || binoptab == umul_widen_optab || binoptab == smul_highpart_optab || binoptab == umul_highpart_optab); } -/* X is to be used in mode MODE as an operand to BINOPTAB. If we're +/* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're optimizing, and if the operand is a constant that costs more than 1 instruction, force the constant into a register and return that register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */ static rtx -avoid_expensive_constant (enum machine_mode mode, optab binoptab, - rtx x, bool unsignedp) +avoid_expensive_constant (machine_mode mode, optab binoptab, + int opn, rtx x, bool unsignedp) { bool speed = optimize_insn_for_speed_p (); if (mode != VOIDmode && optimize && CONSTANT_P (x) - && rtx_cost (x, binoptab->code, speed) > rtx_cost (x, SET, speed)) + && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed) + > set_src_cost (x, mode, speed))) { if (CONST_INT_P (x)) { @@ -1384,42 +983,40 @@ is an insn that directly implements the indicated operation. Returns null if this is not possible. */ static rtx -expand_binop_directly (enum machine_mode mode, optab binoptab, +expand_binop_directly (machine_mode mode, optab binoptab, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods, - rtx last) + rtx_insn *last) { - int icode = (int) optab_handler (binoptab, mode); - enum machine_mode mode0 = insn_data[icode].operand[1].mode; - enum machine_mode mode1 = insn_data[icode].operand[2].mode; - enum machine_mode tmp_mode; + machine_mode from_mode = widened_mode (mode, op0, op1); + enum insn_code icode = find_widening_optab_handler (binoptab, mode, + from_mode, 1); + machine_mode xmode0 = insn_data[(int) icode].operand[1].mode; + machine_mode xmode1 = insn_data[(int) icode].operand[2].mode; + machine_mode mode0, mode1, tmp_mode; + struct expand_operand ops[3]; bool commutative_p; - rtx pat; + rtx_insn *pat; rtx xop0 = op0, xop1 = op1; - rtx temp; - rtx swap; - - if (target) - temp = target; - else - temp = gen_reg_rtx (mode); + bool canonicalize_op1 = false; /* If it is a commutative operator and the modes would match if we would swap the operands, we can save the conversions. */ commutative_p = commutative_optab_p (binoptab); if (commutative_p - && GET_MODE (xop0) != mode0 && GET_MODE (xop1) != mode1 - && GET_MODE (xop0) == mode1 && GET_MODE (xop1) == mode1) - { - swap = xop0; - xop0 = xop1; - xop1 = swap; - } + && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1 + && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1) + std::swap (xop0, xop1); /* If we are optimizing, force expensive constants into a register. */ - xop0 = avoid_expensive_constant (mode0, binoptab, xop0, unsignedp); + xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp); if (!shift_optab_p (binoptab)) - xop1 = avoid_expensive_constant (mode1, binoptab, xop1, unsignedp); + xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp); + else + /* Shifts and rotates often use a different mode for op1 from op0; + for VOIDmode constants we don't know the mode, so force it + to be canonicalized using convert_modes. */ + canonicalize_op1 = true; /* In case the insn wants input operands in modes different from those of the actual operands, convert the operands. It would @@ -1427,19 +1024,20 @@ that they're properly zero-extended, sign-extended or truncated for their mode. */ - if (GET_MODE (xop0) != mode0 && mode0 != VOIDmode) - xop0 = convert_modes (mode0, - GET_MODE (xop0) != VOIDmode - ? GET_MODE (xop0) - : mode, - xop0, unsignedp); - - if (GET_MODE (xop1) != mode1 && mode1 != VOIDmode) - xop1 = convert_modes (mode1, - GET_MODE (xop1) != VOIDmode - ? GET_MODE (xop1) - : mode, - xop1, unsignedp); + mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode; + if (xmode0 != VOIDmode && xmode0 != mode0) + { + xop0 = convert_modes (xmode0, mode0, xop0, unsignedp); + mode0 = xmode0; + } + + mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1) + ? GET_MODE (xop1) : mode); + if (xmode1 != VOIDmode && xmode1 != mode1) + { + xop1 = convert_modes (xmode1, mode1, xop1, unsignedp); + mode1 = xmode1; + } /* If operation is commutative, try to make the first operand a register. @@ -1447,23 +1045,11 @@ Also try to make the last operand a constant. */ if (commutative_p && swap_commutative_operands_with_target (target, xop0, xop1)) - { - swap = xop1; - xop1 = xop0; - xop0 = swap; - } + std::swap (xop0, xop1); /* Now, if insn's predicates don't allow our operands, put them into pseudo regs. */ - if (!insn_data[icode].operand[1].predicate (xop0, mode0) - && mode0 != VOIDmode) - xop0 = copy_to_mode_reg (mode0, xop0); - - if (!insn_data[icode].operand[2].predicate (xop1, mode1) - && mode1 != VOIDmode) - xop1 = copy_to_mode_reg (mode1, xop1); - if (binoptab == vec_pack_trunc_optab || binoptab == vec_pack_usat_optab || binoptab == vec_pack_ssat_optab @@ -1472,24 +1058,30 @@ { /* The mode of the result is different then the mode of the arguments. */ - tmp_mode = insn_data[icode].operand[0].mode; - if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode)) - return 0; + tmp_mode = insn_data[(int) icode].operand[0].mode; + if (VECTOR_MODE_P (mode) + && GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode)) + { + delete_insns_since (last); + return NULL_RTX; + } } else tmp_mode = mode; - if (!insn_data[icode].operand[0].predicate (temp, tmp_mode)) - temp = gen_reg_rtx (tmp_mode); - - pat = GEN_FCN (icode) (temp, xop0, xop1); + create_output_operand (&ops[0], target, tmp_mode); + create_input_operand (&ops[1], xop0, mode0); + create_input_operand (&ops[2], xop1, mode1); + pat = maybe_gen_insn (icode, 3, ops); if (pat) { /* If PAT is composed of more than one insn, try to add an appropriate REG_EQUAL note to it. If we can't because TEMP conflicts with an operand, call expand_binop again, this time without a target. */ if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX - && ! add_equal_note (pat, temp, binoptab->code, xop0, xop1)) + && ! add_equal_note (pat, ops[0].value, + optab_to_code (binoptab), + ops[1].value, ops[2].value)) { delete_insns_since (last); return expand_binop (mode, binoptab, op0, op1, NULL_RTX, @@ -1497,9 +1089,8 @@ } emit_insn (pat); - return temp; - } - + return ops[0].value; + } delete_insns_since (last); return NULL_RTX; } @@ -1516,18 +1107,19 @@ this may or may not be TARGET. */ rtx -expand_binop (enum machine_mode mode, optab binoptab, rtx op0, rtx op1, +expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods) { enum optab_methods next_methods = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN ? OPTAB_WIDEN : methods); enum mode_class mclass; - enum machine_mode wider_mode; + machine_mode wider_mode; + scalar_int_mode int_mode; rtx libfunc; rtx temp; - rtx entry_last = get_last_insn (); - rtx last; + rtx_insn *entry_last = get_last_insn (); + rtx_insn *last; mclass = GET_MODE_CLASS (mode); @@ -1539,6 +1131,16 @@ op1 = negate_rtx (mode, op1); binoptab = add_optab; } + /* For shifts, constant invalid op1 might be expanded from different + mode than MODE. As those are invalid, force them to a register + to avoid further problems during expansion. */ + else if (CONST_INT_P (op1) + && shift_optab_p (binoptab) + && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode))) + { + op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode)); + op1 = force_reg (GET_MODE_INNER (mode), op1); + } /* Record where to delete back to if we backtrack. */ last = get_last_insn (); @@ -1546,7 +1148,9 @@ /* If we can do it with a three-operand insn, do so. */ if (methods != OPTAB_MUST_WIDEN - && optab_handler (binoptab, mode) != CODE_FOR_nothing) + && find_widening_optab_handler (binoptab, mode, + widened_mode (mode, op0, op1), 1) + != CODE_FOR_nothing) { temp = expand_binop_directly (mode, binoptab, op0, op1, target, unsignedp, methods, last); @@ -1560,22 +1164,22 @@ && optab_handler (rotr_optab, mode) != CODE_FOR_nothing) || (binoptab == rotr_optab && optab_handler (rotl_optab, mode) != CODE_FOR_nothing)) - && mclass == MODE_INT) + && is_int_mode (mode, &int_mode)) { optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab); rtx newop1; - unsigned int bits = GET_MODE_BITSIZE (mode); + unsigned int bits = GET_MODE_PRECISION (int_mode); if (CONST_INT_P (op1)) newop1 = GEN_INT (bits - INTVAL (op1)); - else if (targetm.shift_truncation_mask (mode) == bits - 1) + else if (targetm.shift_truncation_mask (int_mode) == bits - 1) newop1 = negate_rtx (GET_MODE (op1), op1); else newop1 = expand_binop (GET_MODE (op1), sub_optab, - GEN_INT (bits), op1, + gen_int_mode (bits, GET_MODE (op1)), op1, NULL_RTX, unsignedp, OPTAB_DIRECT); - temp = expand_binop_directly (mode, otheroptab, op0, newop1, + temp = expand_binop_directly (int_mode, otheroptab, op0, newop1, target, unsignedp, methods, last); if (temp) return temp; @@ -1585,42 +1189,81 @@ takes operands of this mode and makes a wider mode. */ if (binoptab == smul_optab - && GET_MODE_WIDER_MODE (mode) != VOIDmode - && (optab_handler ((unsignedp ? umul_widen_optab : smul_widen_optab), - GET_MODE_WIDER_MODE (mode)) - != CODE_FOR_nothing)) - { - temp = expand_binop (GET_MODE_WIDER_MODE (mode), + && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode) + && (convert_optab_handler ((unsignedp + ? umul_widen_optab + : smul_widen_optab), + wider_mode, mode) != CODE_FOR_nothing)) + { + temp = expand_binop (wider_mode, unsignedp ? umul_widen_optab : smul_widen_optab, op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT); if (temp != 0) { if (GET_MODE_CLASS (mode) == MODE_INT - && TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (mode), - GET_MODE_BITSIZE (GET_MODE (temp)))) + && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp))) return gen_lowpart (mode, temp); else return convert_to_mode (mode, temp, unsignedp); } } + /* If this is a vector shift by a scalar, see if we can do a vector + shift by a vector. If so, broadcast the scalar into a vector. */ + if (mclass == MODE_VECTOR_INT) + { + optab otheroptab = unknown_optab; + + if (binoptab == ashl_optab) + otheroptab = vashl_optab; + else if (binoptab == ashr_optab) + otheroptab = vashr_optab; + else if (binoptab == lshr_optab) + otheroptab = vlshr_optab; + else if (binoptab == rotl_optab) + otheroptab = vrotl_optab; + else if (binoptab == rotr_optab) + otheroptab = vrotr_optab; + + if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing) + { + /* The scalar may have been extended to be too wide. Truncate + it back to the proper size to fit in the broadcast vector. */ + scalar_mode inner_mode = GET_MODE_INNER (mode); + if (!CONST_INT_P (op1) + && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1))) + > GET_MODE_BITSIZE (inner_mode))) + op1 = force_reg (inner_mode, + simplify_gen_unary (TRUNCATE, inner_mode, op1, + GET_MODE (op1))); + rtx vop1 = expand_vector_broadcast (mode, op1); + if (vop1) + { + temp = expand_binop_directly (mode, otheroptab, op0, vop1, + target, unsignedp, methods, last); + if (temp) + return temp; + } + } + } + /* Look for a wider mode of the same class for which we think we can open-code the operation. Check for a widening multiply at the wider mode as well. */ if (CLASS_HAS_WIDER_MODES_P (mclass) && methods != OPTAB_DIRECT && methods != OPTAB_LIB) - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { + machine_mode next_mode; if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing || (binoptab == smul_optab - && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode - && (optab_handler ((unsignedp ? umul_widen_optab - : smul_widen_optab), - GET_MODE_WIDER_MODE (wider_mode)) + && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode) + && (find_widening_optab_handler ((unsignedp + ? umul_widen_optab + : smul_widen_optab), + next_mode, mode, 0) != CODE_FOR_nothing))) { rtx xop0 = op0, xop1 = op1; @@ -1637,10 +1280,10 @@ && mclass == MODE_INT) { no_extend = 1; - xop0 = avoid_expensive_constant (mode, binoptab, + xop0 = avoid_expensive_constant (mode, binoptab, 0, xop0, unsignedp); if (binoptab != ashl_optab) - xop1 = avoid_expensive_constant (mode, binoptab, + xop1 = avoid_expensive_constant (mode, binoptab, 1, xop1, unsignedp); } @@ -1655,8 +1298,7 @@ if (temp) { if (mclass != MODE_INT - || !TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (mode), - GET_MODE_BITSIZE (wider_mode))) + || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) { if (target == 0) target = gen_reg_rtx (mode); @@ -1677,35 +1319,34 @@ Also try to make the last operand a constant. */ if (commutative_optab_p (binoptab) && swap_commutative_operands_with_target (target, op0, op1)) - { - temp = op1; - op1 = op0; - op0 = temp; - } + std::swap (op0, op1); /* These can be done a word at a time. */ if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab) - && mclass == MODE_INT - && GET_MODE_SIZE (mode) > UNITS_PER_WORD + && is_int_mode (mode, &int_mode) + && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) { int i; - rtx insns; + rtx_insn *insns; /* If TARGET is the same as one of the operands, the REG_EQUAL note won't be accurate, so use a new target. */ - if (target == 0 || target == op0 || target == op1) - target = gen_reg_rtx (mode); + if (target == 0 + || target == op0 + || target == op1 + || !valid_multiword_target_p (target)) + target = gen_reg_rtx (int_mode); start_sequence (); /* Do the actual arithmetic. */ - for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++) + for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++) { - rtx target_piece = operand_subword (target, i, 1, mode); + rtx target_piece = operand_subword (target, i, 1, int_mode); rtx x = expand_binop (word_mode, binoptab, - operand_subword_force (op0, i, mode), - operand_subword_force (op1, i, mode), + operand_subword_force (op0, i, int_mode), + operand_subword_force (op1, i, int_mode), target_piece, unsignedp, next_methods); if (x == 0) @@ -1718,7 +1359,7 @@ insns = get_insns (); end_sequence (); - if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD) + if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD) { emit_insn (insns); return target; @@ -1728,19 +1369,22 @@ /* Synthesize double word shifts from single word shifts. */ if ((binoptab == lshr_optab || binoptab == ashl_optab || binoptab == ashr_optab) - && mclass == MODE_INT + && is_int_mode (mode, &int_mode) && (CONST_INT_P (op1) || optimize_insn_for_speed_p ()) - && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD + && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD + && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode) && optab_handler (binoptab, word_mode) != CODE_FOR_nothing && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) { unsigned HOST_WIDE_INT shift_mask, double_shift_mask; - enum machine_mode op1_mode; - - double_shift_mask = targetm.shift_truncation_mask (mode); + scalar_int_mode op1_mode; + + double_shift_mask = targetm.shift_truncation_mask (int_mode); shift_mask = targetm.shift_truncation_mask (word_mode); - op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode; + op1_mode = (GET_MODE (op1) != VOIDmode + ? as_a <scalar_int_mode> (GET_MODE (op1)) + : word_mode); /* Apply the truncation to constant shifts. */ if (double_shift_mask > 0 && CONST_INT_P (op1)) @@ -1755,15 +1399,18 @@ || (shift_mask == BITS_PER_WORD - 1 && double_shift_mask == BITS_PER_WORD * 2 - 1)) { - rtx insns; + rtx_insn *insns; rtx into_target, outof_target; rtx into_input, outof_input; int left_shift, outof_word; /* If TARGET is the same as one of the operands, the REG_EQUAL note won't be accurate, so use a new target. */ - if (target == 0 || target == op0 || target == op1) - target = gen_reg_rtx (mode); + if (target == 0 + || target == op0 + || target == op1 + || !valid_multiword_target_p (target)) + target = gen_reg_rtx (int_mode); start_sequence (); @@ -1775,11 +1422,11 @@ left_shift = binoptab == ashl_optab; outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; - outof_target = operand_subword (target, outof_word, 1, mode); - into_target = operand_subword (target, 1 - outof_word, 1, mode); - - outof_input = operand_subword_force (op0, outof_word, mode); - into_input = operand_subword_force (op0, 1 - outof_word, mode); + outof_target = operand_subword (target, outof_word, 1, int_mode); + into_target = operand_subword (target, 1 - outof_word, 1, int_mode); + + outof_input = operand_subword_force (op0, outof_word, int_mode); + into_input = operand_subword_force (op0, 1 - outof_word, int_mode); if (expand_doubleword_shift (op1_mode, binoptab, outof_input, into_input, op1, @@ -1798,13 +1445,13 @@ /* Synthesize double word rotates from single word shifts. */ if ((binoptab == rotl_optab || binoptab == rotr_optab) - && mclass == MODE_INT + && is_int_mode (mode, &int_mode) && CONST_INT_P (op1) - && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD + && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) { - rtx insns; + rtx_insn *insns; rtx into_target, outof_target; rtx into_input, outof_input; rtx inter; @@ -1816,8 +1463,12 @@ opportunities, and second because if target and op0 happen to be MEMs designating the same location, we would risk clobbering it too early in the code sequence we generate below. */ - if (target == 0 || target == op0 || target == op1 || ! REG_P (target)) - target = gen_reg_rtx (mode); + if (target == 0 + || target == op0 + || target == op1 + || !REG_P (target) + || !valid_multiword_target_p (target)) + target = gen_reg_rtx (int_mode); start_sequence (); @@ -1831,11 +1482,11 @@ left_shift = (binoptab == rotl_optab); outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; - outof_target = operand_subword (target, outof_word, 1, mode); - into_target = operand_subword (target, 1 - outof_word, 1, mode); - - outof_input = operand_subword_force (op0, outof_word, mode); - into_input = operand_subword_force (op0, 1 - outof_word, mode); + outof_target = operand_subword (target, outof_word, 1, int_mode); + into_target = operand_subword (target, 1 - outof_word, 1, int_mode); + + outof_input = operand_subword_force (op0, outof_word, int_mode); + into_input = operand_subword_force (op0, 1 - outof_word, int_mode); if (shift_count == BITS_PER_WORD) { @@ -1911,13 +1562,13 @@ /* These can be done a word at a time by propagating carries. */ if ((binoptab == add_optab || binoptab == sub_optab) - && mclass == MODE_INT - && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD + && is_int_mode (mode, &int_mode) + && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) { unsigned int i; optab otheroptab = binoptab == add_optab ? sub_optab : add_optab; - const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD; + const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; rtx carry_in = NULL_RTX, carry_out = NULL_RTX; rtx xop0, xop1, xtarget; @@ -1931,12 +1582,12 @@ #endif /* Prepare the operands. */ - xop0 = force_reg (mode, op0); - xop1 = force_reg (mode, op1); - - xtarget = gen_reg_rtx (mode); - - if (target == 0 || !REG_P (target)) + xop0 = force_reg (int_mode, op0); + xop1 = force_reg (int_mode, op1); + + xtarget = gen_reg_rtx (int_mode); + + if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target)) target = xtarget; /* Indicate for flow that the entire target reg is being set. */ @@ -1947,9 +1598,9 @@ for (i = 0; i < nwords; i++) { int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i); - rtx target_piece = operand_subword (xtarget, index, 1, mode); - rtx op0_piece = operand_subword_force (xop0, index, mode); - rtx op1_piece = operand_subword_force (xop1, index, mode); + rtx target_piece = operand_subword (xtarget, index, 1, int_mode); + rtx op0_piece = operand_subword_force (xop0, index, int_mode); + rtx op1_piece = operand_subword_force (xop1, index, int_mode); rtx x; /* Main add/subtract of the input operands. */ @@ -2008,18 +1659,18 @@ carry_in = carry_out; } - if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD) + if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD) { - if (optab_handler (mov_optab, mode) != CODE_FOR_nothing + if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing || ! rtx_equal_p (target, xtarget)) { - rtx temp = emit_move_insn (target, xtarget); - - set_unique_reg_note (temp, - REG_EQUAL, - gen_rtx_fmt_ee (binoptab->code, mode, - copy_rtx (xop0), - copy_rtx (xop1))); + rtx_insn *temp = emit_move_insn (target, xtarget); + + set_dst_reg_note (temp, REG_EQUAL, + gen_rtx_fmt_ee (optab_to_code (binoptab), + int_mode, copy_rtx (xop0), + copy_rtx (xop1)), + target); } else target = xtarget; @@ -2037,25 +1688,26 @@ try using a signed widening multiply. */ if (binoptab == smul_optab - && mclass == MODE_INT - && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD + && is_int_mode (mode, &int_mode) + && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing && optab_handler (add_optab, word_mode) != CODE_FOR_nothing) { rtx product = NULL_RTX; - - if (optab_handler (umul_widen_optab, mode) != CODE_FOR_nothing) + if (widening_optab_handler (umul_widen_optab, int_mode, word_mode) + != CODE_FOR_nothing) { - product = expand_doubleword_mult (mode, op0, op1, target, + product = expand_doubleword_mult (int_mode, op0, op1, target, true, methods); if (!product) delete_insns_since (last); } if (product == NULL_RTX - && optab_handler (smul_widen_optab, mode) != CODE_FOR_nothing) + && (widening_optab_handler (smul_widen_optab, int_mode, word_mode) + != CODE_FOR_nothing)) { - product = expand_doubleword_mult (mode, op0, op1, target, + product = expand_doubleword_mult (int_mode, op0, op1, target, false, methods); if (!product) delete_insns_since (last); @@ -2063,14 +1715,16 @@ if (product != NULL_RTX) { - if (optab_handler (mov_optab, mode) != CODE_FOR_nothing) + if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing) { - temp = emit_move_insn (target ? target : product, product); - set_unique_reg_note (temp, - REG_EQUAL, - gen_rtx_fmt_ee (MULT, mode, - copy_rtx (op0), - copy_rtx (op1))); + rtx_insn *move = emit_move_insn (target ? target : product, + product); + set_dst_reg_note (move, + REG_EQUAL, + gen_rtx_fmt_ee (MULT, int_mode, + copy_rtx (op0), + copy_rtx (op1)), + target ? target : product); } return product; } @@ -2083,9 +1737,9 @@ if (libfunc && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN)) { - rtx insns; + rtx_insn *insns; rtx op1x = op1; - enum machine_mode op1_mode = mode; + machine_mode op1_mode = mode; rtx value; start_sequence (); @@ -2105,15 +1759,18 @@ /* Pass 1 for NO_QUEUE so we don't lose any increments if the libcall is cse'd or moved. */ value = emit_library_call_value (libfunc, - NULL_RTX, LCT_CONST, mode, 2, + NULL_RTX, LCT_CONST, mode, op0, mode, op1x, op1_mode); insns = get_insns (); end_sequence (); + bool trapv = trapv_binoptab_p (binoptab); target = gen_reg_rtx (mode); - emit_libcall_block (insns, target, value, - gen_rtx_fmt_ee (binoptab->code, mode, op0, op1)); + emit_libcall_block_1 (insns, target, value, + trapv ? NULL_RTX + : gen_rtx_fmt_ee (optab_to_code (binoptab), + mode, op0, op1), trapv); return target; } @@ -2140,11 +1797,10 @@ if (CLASS_HAS_WIDER_MODES_P (mclass)) { - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { - if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing + if (find_widening_optab_handler (binoptab, wider_mode, mode, 1) + != CODE_FOR_nothing || (methods == OPTAB_LIB && optab_libfunc (binoptab, wider_mode))) { @@ -2174,8 +1830,7 @@ if (temp) { if (mclass != MODE_INT - || !TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (mode), - GET_MODE_BITSIZE (wider_mode))) + || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) { if (target == 0) target = gen_reg_rtx (mode); @@ -2203,13 +1858,13 @@ of an unsigned wider operation, since the result would be the same. */ rtx -sign_expand_binop (enum machine_mode mode, optab uoptab, optab soptab, +sign_expand_binop (machine_mode mode, optab uoptab, optab soptab, rtx op0, rtx op1, rtx target, int unsignedp, enum optab_methods methods) { rtx temp; optab direct_optab = unsignedp ? uoptab : soptab; - struct optab_d wide_soptab; + bool save_enable; /* Do it without widening, if possible. */ temp = expand_binop (mode, direct_optab, op0, op1, target, @@ -2217,38 +1872,38 @@ if (temp || methods == OPTAB_DIRECT) return temp; - /* Try widening to a signed int. Make a fake signed optab that - hides any signed insn for direct use. */ - wide_soptab = *soptab; - set_optab_handler (&wide_soptab, mode, CODE_FOR_nothing); - /* We don't want to generate new hash table entries from this fake - optab. */ - wide_soptab.libcall_gen = NULL; - - temp = expand_binop (mode, &wide_soptab, op0, op1, target, + /* Try widening to a signed int. Disable any direct use of any + signed insn in the current mode. */ + save_enable = swap_optab_enable (soptab, mode, false); + + temp = expand_binop (mode, soptab, op0, op1, target, unsignedp, OPTAB_WIDEN); /* For unsigned operands, try widening to an unsigned int. */ - if (temp == 0 && unsignedp) + if (!temp && unsignedp) temp = expand_binop (mode, uoptab, op0, op1, target, unsignedp, OPTAB_WIDEN); if (temp || methods == OPTAB_WIDEN) - return temp; + goto egress; /* Use the right width libcall if that exists. */ - temp = expand_binop (mode, direct_optab, op0, op1, target, unsignedp, OPTAB_LIB); + temp = expand_binop (mode, direct_optab, op0, op1, target, + unsignedp, OPTAB_LIB); if (temp || methods == OPTAB_LIB) - return temp; + goto egress; /* Must widen and use a libcall, use either signed or unsigned. */ - temp = expand_binop (mode, &wide_soptab, op0, op1, target, + temp = expand_binop (mode, soptab, op0, op1, target, unsignedp, methods); - if (temp != 0) - return temp; - if (unsignedp) - return expand_binop (mode, uoptab, op0, op1, target, + if (!temp && unsignedp) + temp = expand_binop (mode, uoptab, op0, op1, target, unsignedp, methods); - return 0; + + egress: + /* Undo the fiddling above. */ + if (save_enable) + swap_optab_enable (soptab, mode, true); + return temp; } /* Generate code to perform an operation specified by UNOPPTAB @@ -2266,11 +1921,11 @@ expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1, int unsignedp) { - enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); + machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); enum mode_class mclass; - enum machine_mode wider_mode; - rtx entry_last = get_last_insn (); - rtx last; + machine_mode wider_mode; + rtx_insn *entry_last = get_last_insn (); + rtx_insn *last; mclass = GET_MODE_CLASS (mode); @@ -2284,41 +1939,21 @@ if (optab_handler (unoptab, mode) != CODE_FOR_nothing) { - int icode = (int) optab_handler (unoptab, mode); - enum machine_mode mode0 = insn_data[icode].operand[2].mode; - rtx pat; - rtx xop0 = op0; - - if (GET_MODE (xop0) != VOIDmode - && GET_MODE (xop0) != mode0) - xop0 = convert_to_mode (mode0, xop0, unsignedp); - - /* Now, if insn doesn't accept these operands, put them into pseudos. */ - if (!insn_data[icode].operand[2].predicate (xop0, mode0)) - xop0 = copy_to_mode_reg (mode0, xop0); - - /* We could handle this, but we should always be called with a pseudo - for our targets and all insns should take them as outputs. */ - gcc_assert (insn_data[icode].operand[0].predicate (targ0, mode)); - gcc_assert (insn_data[icode].operand[1].predicate (targ1, mode)); - - pat = GEN_FCN (icode) (targ0, targ1, xop0); - if (pat) - { - emit_insn (pat); - return 1; - } - else - delete_insns_since (last); + struct expand_operand ops[3]; + enum insn_code icode = optab_handler (unoptab, mode); + + create_fixed_operand (&ops[0], targ0); + create_fixed_operand (&ops[1], targ1); + create_convert_operand_from (&ops[2], op0, mode, unsignedp); + if (maybe_expand_insn (icode, 3, ops)) + return 1; } /* It can't be done in this mode. Can we do it in a wider mode? */ if (CLASS_HAS_WIDER_MODES_P (mclass)) { - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) { @@ -2358,11 +1993,11 @@ expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1, int unsignedp) { - enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); + machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); enum mode_class mclass; - enum machine_mode wider_mode; - rtx entry_last = get_last_insn (); - rtx last; + machine_mode wider_mode; + rtx_insn *entry_last = get_last_insn (); + rtx_insn *last; mclass = GET_MODE_CLASS (mode); @@ -2376,65 +2011,30 @@ if (optab_handler (binoptab, mode) != CODE_FOR_nothing) { - int icode = (int) optab_handler (binoptab, mode); - enum machine_mode mode0 = insn_data[icode].operand[1].mode; - enum machine_mode mode1 = insn_data[icode].operand[2].mode; - rtx pat; + struct expand_operand ops[4]; + enum insn_code icode = optab_handler (binoptab, mode); + machine_mode mode0 = insn_data[icode].operand[1].mode; + machine_mode mode1 = insn_data[icode].operand[2].mode; rtx xop0 = op0, xop1 = op1; /* If we are optimizing, force expensive constants into a register. */ - xop0 = avoid_expensive_constant (mode0, binoptab, xop0, unsignedp); - xop1 = avoid_expensive_constant (mode1, binoptab, xop1, unsignedp); - - /* In case the insn wants input operands in modes different from - those of the actual operands, convert the operands. It would - seem that we don't need to convert CONST_INTs, but we do, so - that they're properly zero-extended, sign-extended or truncated - for their mode. */ - - if (GET_MODE (op0) != mode0 && mode0 != VOIDmode) - xop0 = convert_modes (mode0, - GET_MODE (op0) != VOIDmode - ? GET_MODE (op0) - : mode, - xop0, unsignedp); - - if (GET_MODE (op1) != mode1 && mode1 != VOIDmode) - xop1 = convert_modes (mode1, - GET_MODE (op1) != VOIDmode - ? GET_MODE (op1) - : mode, - xop1, unsignedp); - - /* Now, if insn doesn't accept these operands, put them into pseudos. */ - if (!insn_data[icode].operand[1].predicate (xop0, mode0)) - xop0 = copy_to_mode_reg (mode0, xop0); - - if (!insn_data[icode].operand[2].predicate (xop1, mode1)) - xop1 = copy_to_mode_reg (mode1, xop1); - - /* We could handle this, but we should always be called with a pseudo - for our targets and all insns should take them as outputs. */ - gcc_assert (insn_data[icode].operand[0].predicate (targ0, mode)); - gcc_assert (insn_data[icode].operand[3].predicate (targ1, mode)); - - pat = GEN_FCN (icode) (targ0, xop0, xop1, targ1); - if (pat) - { - emit_insn (pat); - return 1; - } - else - delete_insns_since (last); + xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp); + xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp); + + create_fixed_operand (&ops[0], targ0); + create_convert_operand_from (&ops[1], op0, mode, unsignedp); + create_convert_operand_from (&ops[2], op1, mode, unsignedp); + create_fixed_operand (&ops[3], targ1); + if (maybe_expand_insn (icode, 4, ops)) + return 1; + delete_insns_since (last); } /* It can't be done in this mode. Can we do it in a wider mode? */ if (CLASS_HAS_WIDER_MODES_P (mclass)) { - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing) { @@ -2473,10 +2073,10 @@ expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1, enum rtx_code code) { - enum machine_mode mode; - enum machine_mode libval_mode; + machine_mode mode; + machine_mode libval_mode; rtx libval; - rtx insns; + rtx_insn *insns; rtx libfunc; /* Exactly one of TARG0 or TARG1 should be non-NULL. */ @@ -2489,11 +2089,10 @@ /* The value returned by the library function will have twice as many bits as the nominal MODE. */ - libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode), - MODE_INT); + libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode)); start_sequence (); libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, - libval_mode, 2, + libval_mode, op0, mode, op1, mode); /* Get the part of VAL containing the value that we want. */ @@ -2513,10 +2112,10 @@ the operation to perform, not an optab pointer. All other arguments are the same. */ rtx -expand_simple_unop (enum machine_mode mode, enum rtx_code code, rtx op0, +expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0, rtx target, int unsignedp) { - optab unop = code_to_optab[(int) code]; + optab unop = code_to_optab (code); gcc_assert (unop); return expand_unop (mode, unop, op0, target, unsignedp); @@ -2525,38 +2124,41 @@ /* Try calculating (clz:narrow x) as - (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)). */ + (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)). + + A similar operation can be used for clrsb. UNOPTAB says which operation + we are trying to expand. */ static rtx -widen_clz (enum machine_mode mode, rtx op0, rtx target) +widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab) { - enum mode_class mclass = GET_MODE_CLASS (mode); - if (CLASS_HAS_WIDER_MODES_P (mclass)) - { - enum machine_mode wider_mode; - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + opt_scalar_int_mode wider_mode_iter; + FOR_EACH_WIDER_MODE (wider_mode_iter, mode) + { + scalar_int_mode wider_mode = wider_mode_iter.require (); + if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) { - if (optab_handler (clz_optab, wider_mode) != CODE_FOR_nothing) - { - rtx xop0, temp, last; - - last = get_last_insn (); - - if (target == 0) - target = gen_reg_rtx (mode); - xop0 = widen_operand (op0, wider_mode, mode, true, false); - temp = expand_unop (wider_mode, clz_optab, xop0, NULL_RTX, true); - if (temp != 0) - temp = expand_binop (wider_mode, sub_optab, temp, - GEN_INT (GET_MODE_BITSIZE (wider_mode) - - GET_MODE_BITSIZE (mode)), - target, true, OPTAB_DIRECT); - if (temp == 0) - delete_insns_since (last); - - return temp; - } + rtx xop0, temp; + rtx_insn *last; + + last = get_last_insn (); + + if (target == 0) + target = gen_reg_rtx (mode); + xop0 = widen_operand (op0, wider_mode, mode, + unoptab != clrsb_optab, false); + temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, + unoptab != clrsb_optab); + if (temp != 0) + temp = expand_binop + (wider_mode, sub_optab, temp, + gen_int_mode (GET_MODE_PRECISION (wider_mode) + - GET_MODE_PRECISION (mode), + wider_mode), + target, true, OPTAB_DIRECT); + if (temp == 0) + delete_insns_since (last); + + return temp; } } return 0; @@ -2565,14 +2167,15 @@ /* Try calculating clz of a double-word quantity as two clz's of word-sized quantities, choosing which based on whether the high word is nonzero. */ static rtx -expand_doubleword_clz (enum machine_mode mode, rtx op0, rtx target) +expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target) { rtx xop0 = force_reg (mode, op0); rtx subhi = gen_highpart (word_mode, xop0); rtx sublo = gen_lowpart (word_mode, xop0); - rtx hi0_label = gen_label_rtx (); - rtx after_label = gen_label_rtx (); - rtx seq, temp, result; + rtx_code_label *hi0_label = gen_label_rtx (); + rtx_code_label *after_label = gen_label_rtx (); + rtx_insn *seq; + rtx temp, result; /* If we were not given a target, use a word_mode register, not a 'mode' register. The result will fit, and nobody is expecting @@ -2599,7 +2202,7 @@ if (temp != result) convert_move (result, temp, true); - emit_jump_insn (gen_jump (after_label)); + emit_jump_insn (targetm.gen_jump (after_label)); emit_barrier (); /* Else clz of the full value is clz of the low word plus the number @@ -2610,7 +2213,7 @@ if (!temp) goto fail; temp = expand_binop (word_mode, add_optab, temp, - GEN_INT (GET_MODE_BITSIZE (word_mode)), + gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode), result, true, OPTAB_DIRECT); if (!temp) goto fail; @@ -2632,37 +2235,89 @@ return 0; } +/* Try calculating popcount of a double-word quantity as two popcount's of + word-sized quantities and summing up the results. */ +static rtx +expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target) +{ + rtx t0, t1, t; + rtx_insn *seq; + + start_sequence (); + + t0 = expand_unop_direct (word_mode, popcount_optab, + operand_subword_force (op0, 0, mode), NULL_RTX, + true); + t1 = expand_unop_direct (word_mode, popcount_optab, + operand_subword_force (op0, 1, mode), NULL_RTX, + true); + if (!t0 || !t1) + { + end_sequence (); + return NULL_RTX; + } + + /* If we were not given a target, use a word_mode register, not a + 'mode' register. The result will fit, and nobody is expecting + anything bigger (the return type of __builtin_popcount* is int). */ + if (!target) + target = gen_reg_rtx (word_mode); + + t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT); + + seq = get_insns (); + end_sequence (); + + add_equal_note (seq, t, POPCOUNT, op0, 0); + emit_insn (seq); + return t; +} + +/* Try calculating + (parity:wide x) + as + (parity:narrow (low (x) ^ high (x))) */ +static rtx +expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target) +{ + rtx t = expand_binop (word_mode, xor_optab, + operand_subword_force (op0, 0, mode), + operand_subword_force (op0, 1, mode), + NULL_RTX, 0, OPTAB_DIRECT); + return expand_unop (word_mode, parity_optab, t, target, true); +} + /* Try calculating (bswap:narrow x) as (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */ static rtx -widen_bswap (enum machine_mode mode, rtx op0, rtx target) +widen_bswap (scalar_int_mode mode, rtx op0, rtx target) { - enum mode_class mclass = GET_MODE_CLASS (mode); - enum machine_mode wider_mode; - rtx x, last; - - if (!CLASS_HAS_WIDER_MODES_P (mclass)) + rtx x; + rtx_insn *last; + opt_scalar_int_mode wider_mode_iter; + + FOR_EACH_WIDER_MODE (wider_mode_iter, mode) + if (optab_handler (bswap_optab, wider_mode_iter.require ()) + != CODE_FOR_nothing) + break; + + if (!wider_mode_iter.exists ()) return NULL_RTX; - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) - if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing) - goto found; - return NULL_RTX; - - found: + scalar_int_mode wider_mode = wider_mode_iter.require (); last = get_last_insn (); x = widen_operand (op0, wider_mode, mode, true, true); x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true); + gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode) + && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)); if (x != 0) x = expand_shift (RSHIFT_EXPR, wider_mode, x, - size_int (GET_MODE_BITSIZE (wider_mode) - - GET_MODE_BITSIZE (mode)), + GET_MODE_BITSIZE (wider_mode) + - GET_MODE_BITSIZE (mode), NULL_RTX, true); if (x != 0) @@ -2680,7 +2335,7 @@ /* Try calculating bswap as two bswaps of two word-sized operands. */ static rtx -expand_doubleword_bswap (enum machine_mode mode, rtx op, rtx target) +expand_doubleword_bswap (machine_mode mode, rtx op, rtx target) { rtx t0, t1; @@ -2689,7 +2344,7 @@ t0 = expand_unop (word_mode, bswap_optab, operand_subword_force (op, 1, mode), NULL_RTX, true); - if (target == 0) + if (target == 0 || !valid_multiword_target_p (target)) target = gen_reg_rtx (mode); if (REG_P (target)) emit_clobber (target); @@ -2702,41 +2357,47 @@ /* Try calculating (parity x) as (and (popcount x) 1), where popcount can also be done in a wider mode. */ static rtx -expand_parity (enum machine_mode mode, rtx op0, rtx target) +expand_parity (scalar_int_mode mode, rtx op0, rtx target) { enum mode_class mclass = GET_MODE_CLASS (mode); - if (CLASS_HAS_WIDER_MODES_P (mclass)) - { - enum machine_mode wider_mode; - for (wider_mode = mode; wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + opt_scalar_int_mode wider_mode_iter; + FOR_EACH_MODE_FROM (wider_mode_iter, mode) + { + scalar_int_mode wider_mode = wider_mode_iter.require (); + if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing) { - if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing) + rtx xop0, temp; + rtx_insn *last; + + last = get_last_insn (); + + if (target == 0 || GET_MODE (target) != wider_mode) + target = gen_reg_rtx (wider_mode); + + xop0 = widen_operand (op0, wider_mode, mode, true, false); + temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX, + true); + if (temp != 0) + temp = expand_binop (wider_mode, and_optab, temp, const1_rtx, + target, true, OPTAB_DIRECT); + + if (temp) { - rtx xop0, temp, last; - - last = get_last_insn (); - - if (target == 0) - target = gen_reg_rtx (mode); - xop0 = widen_operand (op0, wider_mode, mode, true, false); - temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX, - true); - if (temp != 0) - temp = expand_binop (wider_mode, and_optab, temp, const1_rtx, - target, true, OPTAB_DIRECT); - if (temp == 0) - delete_insns_since (last); - - return temp; + if (mclass != MODE_INT + || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) + return convert_to_mode (mode, temp, 0); + else + return gen_lowpart (mode, temp); } + else + delete_insns_since (last); } } return 0; } /* Try calculating ctz(x) as K - clz(x & -x) , - where K is GET_MODE_BITSIZE(mode) - 1. + where K is GET_MODE_PRECISION(mode) - 1. Both __builtin_ctz and __builtin_clz are undefined at zero, so we don't have to worry about what the hardware does in that case. (If @@ -2748,9 +2409,10 @@ less convenient for expand_ffs anyway. */ static rtx -expand_ctz (enum machine_mode mode, rtx op0, rtx target) +expand_ctz (scalar_int_mode mode, rtx op0, rtx target) { - rtx seq, temp; + rtx_insn *seq; + rtx temp; if (optab_handler (clz_optab, mode) == CODE_FOR_nothing) return 0; @@ -2764,7 +2426,8 @@ if (temp) temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true); if (temp) - temp = expand_binop (mode, sub_optab, GEN_INT (GET_MODE_BITSIZE (mode) - 1), + temp = expand_binop (mode, sub_optab, + gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode), temp, target, true, OPTAB_DIRECT); if (temp == 0) @@ -2789,11 +2452,12 @@ may have an undefined value in that case. If they do not give us a convenient value, we have to generate a test and branch. */ static rtx -expand_ffs (enum machine_mode mode, rtx op0, rtx target) +expand_ffs (scalar_int_mode mode, rtx op0, rtx target) { HOST_WIDE_INT val = 0; bool defined_at_zero = false; - rtx temp, seq; + rtx temp; + rtx_insn *seq; if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing) { @@ -2815,7 +2479,7 @@ if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2) { defined_at_zero = true; - val = (GET_MODE_BITSIZE (mode) - 1) - val; + val = (GET_MODE_PRECISION (mode) - 1) - val; } } else @@ -2835,7 +2499,7 @@ the operation sets condition codes that can be recycled for this. (This is true on i386, for instance.) */ - rtx nonzero_label = gen_label_rtx (); + rtx_code_label *nonzero_label = gen_label_rtx (); emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0, mode, true, nonzero_label); @@ -2845,7 +2509,7 @@ /* temp now has a value in the range -1..bitsize-1. ffs is supposed to produce a value in the range 0..bitsize. */ - temp = expand_binop (mode, add_optab, temp, GEN_INT (1), + temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode), target, false, OPTAB_DIRECT); if (!temp) goto fail; @@ -2868,8 +2532,8 @@ register will work around the situation. */ static rtx -lowpart_subreg_maybe_copy (enum machine_mode omode, rtx val, - enum machine_mode imode) +lowpart_subreg_maybe_copy (machine_mode omode, rtx val, + machine_mode imode) { rtx ret; ret = lowpart_subreg (omode, val, imode); @@ -2886,14 +2550,14 @@ logical operation on the sign bit. */ static rtx -expand_absneg_bit (enum rtx_code code, enum machine_mode mode, +expand_absneg_bit (enum rtx_code code, scalar_float_mode mode, rtx op0, rtx target) { const struct real_format *fmt; int bitpos, word, nwords, i; - enum machine_mode imode; - double_int mask; - rtx temp, insns; + scalar_int_mode imode; + rtx temp; + rtx_insn *insns; /* The format has to have a simple sign bit. */ fmt = REAL_MODE_FORMAT (mode); @@ -2910,8 +2574,7 @@ if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) { - imode = int_mode_for_mode (mode); - if (imode == BLKmode) + if (!int_mode_for_mode (mode).exists (&imode)) return NULL_RTX; word = 0; nwords = 1; @@ -2928,11 +2591,13 @@ nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD; } - mask = double_int_setbit (double_int_zero, bitpos); + wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); if (code == ABS) - mask = double_int_not (mask); - - if (target == 0 || target == op0) + mask = ~mask; + + if (target == 0 + || target == op0 + || (nwords > 1 && !valid_multiword_target_p (target))) target = gen_reg_rtx (mode); if (nwords > 1) @@ -2948,7 +2613,7 @@ { temp = expand_binop (imode, code == ABS ? and_optab : xor_optab, op0_piece, - immed_double_int_const (mask, imode), + immed_wide_int_const (mask, imode), targ_piece, 1, OPTAB_LIB_WIDEN); if (temp != targ_piece) emit_move_insn (targ_piece, temp); @@ -2966,12 +2631,13 @@ { temp = expand_binop (imode, code == ABS ? and_optab : xor_optab, gen_lowpart (imode, op0), - immed_double_int_const (mask, imode), + immed_wide_int_const (mask, imode), gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN); target = lowpart_subreg_maybe_copy (mode, temp, imode); - set_unique_reg_note (get_last_insn (), REG_EQUAL, - gen_rtx_fmt_e (code, mode, copy_rtx (op0))); + set_dst_reg_note (get_last_insn (), REG_EQUAL, + gen_rtx_fmt_e (code, mode, copy_rtx (op0)), + target); } return target; @@ -2980,39 +2646,25 @@ /* As expand_unop, but will fail rather than attempt the operation in a different mode or with a libcall. */ static rtx -expand_unop_direct (enum machine_mode mode, optab unoptab, rtx op0, rtx target, - int unsignedp) +expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target, + int unsignedp) { if (optab_handler (unoptab, mode) != CODE_FOR_nothing) { - int icode = (int) optab_handler (unoptab, mode); - enum machine_mode mode0 = insn_data[icode].operand[1].mode; - rtx xop0 = op0; - rtx last = get_last_insn (); - rtx pat, temp; - - if (target) - temp = target; - else - temp = gen_reg_rtx (mode); - - if (GET_MODE (xop0) != VOIDmode - && GET_MODE (xop0) != mode0) - xop0 = convert_to_mode (mode0, xop0, unsignedp); - - /* Now, if insn doesn't accept our operand, put it into a pseudo. */ - - if (!insn_data[icode].operand[1].predicate (xop0, mode0)) - xop0 = copy_to_mode_reg (mode0, xop0); - - if (!insn_data[icode].operand[0].predicate (temp, mode)) - temp = gen_reg_rtx (mode); - - pat = GEN_FCN (icode) (temp, xop0); + struct expand_operand ops[2]; + enum insn_code icode = optab_handler (unoptab, mode); + rtx_insn *last = get_last_insn (); + rtx_insn *pat; + + create_output_operand (&ops[0], target, mode); + create_convert_operand_from (&ops[1], op0, mode, unsignedp); + pat = maybe_gen_insn (icode, 2, ops); if (pat) { if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX - && ! add_equal_note (pat, temp, unoptab->code, xop0, NULL_RTX)) + && ! add_equal_note (pat, ops[0].value, + optab_to_code (unoptab), + ops[1].value, NULL_RTX)) { delete_insns_since (last); return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp); @@ -3020,10 +2672,8 @@ emit_insn (pat); - return temp; + return ops[0].value; } - else - delete_insns_since (last); } return 0; } @@ -3040,11 +2690,13 @@ this may or may not be TARGET. */ rtx -expand_unop (enum machine_mode mode, optab unoptab, rtx op0, rtx target, +expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target, int unsignedp) { enum mode_class mclass = GET_MODE_CLASS (mode); - enum machine_mode wider_mode; + machine_mode wider_mode; + scalar_int_mode int_mode; + scalar_float_mode float_mode; rtx temp; rtx libfunc; @@ -3057,48 +2709,128 @@ /* Widening (or narrowing) clz needs special treatment. */ if (unoptab == clz_optab) { - temp = widen_clz (mode, op0, target); - if (temp) - return temp; - - if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD - && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) + if (is_a <scalar_int_mode> (mode, &int_mode)) { - temp = expand_doubleword_clz (mode, op0, target); + temp = widen_leading (int_mode, op0, target, unoptab); + if (temp) + return temp; + + if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD + && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) + { + temp = expand_doubleword_clz (int_mode, op0, target); + if (temp) + return temp; + } + } + + goto try_libcall; + } + + if (unoptab == clrsb_optab) + { + if (is_a <scalar_int_mode> (mode, &int_mode)) + { + temp = widen_leading (int_mode, op0, target, unoptab); if (temp) return temp; } - - goto try_libcall; + goto try_libcall; + } + + if (unoptab == popcount_optab + && is_a <scalar_int_mode> (mode, &int_mode) + && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD + && optab_handler (unoptab, word_mode) != CODE_FOR_nothing + && optimize_insn_for_speed_p ()) + { + temp = expand_doubleword_popcount (int_mode, op0, target); + if (temp) + return temp; + } + + if (unoptab == parity_optab + && is_a <scalar_int_mode> (mode, &int_mode) + && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD + && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing + || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing) + && optimize_insn_for_speed_p ()) + { + temp = expand_doubleword_parity (int_mode, op0, target); + if (temp) + return temp; } /* Widening (or narrowing) bswap needs special treatment. */ if (unoptab == bswap_optab) { - temp = widen_bswap (mode, op0, target); - if (temp) - return temp; - - if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD - && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) + /* HImode is special because in this mode BSWAP is equivalent to ROTATE + or ROTATERT. First try these directly; if this fails, then try the + obvious pair of shifts with allowed widening, as this will probably + be always more efficient than the other fallback methods. */ + if (mode == HImode) { - temp = expand_doubleword_bswap (mode, op0, target); + rtx_insn *last; + rtx temp1, temp2; + + if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing) + { + temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target, + unsignedp, OPTAB_DIRECT); + if (temp) + return temp; + } + + if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing) + { + temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target, + unsignedp, OPTAB_DIRECT); + if (temp) + return temp; + } + + last = get_last_insn (); + + temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX, + unsignedp, OPTAB_WIDEN); + temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX, + unsignedp, OPTAB_WIDEN); + if (temp1 && temp2) + { + temp = expand_binop (mode, ior_optab, temp1, temp2, target, + unsignedp, OPTAB_WIDEN); + if (temp) + return temp; + } + + delete_insns_since (last); + } + + if (is_a <scalar_int_mode> (mode, &int_mode)) + { + temp = widen_bswap (int_mode, op0, target); if (temp) return temp; + + if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD + && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) + { + temp = expand_doubleword_bswap (mode, op0, target); + if (temp) + return temp; + } } goto try_libcall; } if (CLASS_HAS_WIDER_MODES_P (mclass)) - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) { rtx xop0 = op0; - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); /* For certain operations, we need not actually extend the narrow operand, as long as we will truncate the @@ -3115,8 +2847,7 @@ if (temp) { if (mclass != MODE_INT - || !TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (mode), - GET_MODE_BITSIZE (wider_mode))) + || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) { if (target == 0) target = gen_reg_rtx (mode); @@ -3133,24 +2864,24 @@ /* These can be done a word at a time. */ if (unoptab == one_cmpl_optab - && mclass == MODE_INT - && GET_MODE_SIZE (mode) > UNITS_PER_WORD + && is_int_mode (mode, &int_mode) + && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) { int i; - rtx insns; - - if (target == 0 || target == op0) - target = gen_reg_rtx (mode); + rtx_insn *insns; + + if (target == 0 || target == op0 || !valid_multiword_target_p (target)) + target = gen_reg_rtx (int_mode); start_sequence (); /* Do the actual arithmetic. */ - for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++) + for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++) { - rtx target_piece = operand_subword (target, i, 1, mode); + rtx target_piece = operand_subword (target, i, 1, int_mode); rtx x = expand_unop (word_mode, unoptab, - operand_subword_force (op0, i, mode), + operand_subword_force (op0, i, int_mode), target_piece, unsignedp); if (target_piece != x) @@ -3164,12 +2895,12 @@ return target; } - if (unoptab->code == NEG) + if (optab_to_code (unoptab) == NEG) { /* Try negating floating point values by flipping the sign bit. */ - if (SCALAR_FLOAT_MODE_P (mode)) + if (is_a <scalar_float_mode> (mode, &float_mode)) { - temp = expand_absneg_bit (NEG, mode, op0, target); + temp = expand_absneg_bit (NEG, float_mode, op0, target); if (temp) return temp; } @@ -3188,25 +2919,25 @@ } /* Try calculating parity (x) as popcount (x) % 2. */ - if (unoptab == parity_optab) - { - temp = expand_parity (mode, op0, target); + if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode)) + { + temp = expand_parity (int_mode, op0, target); if (temp) return temp; } /* Try implementing ffs (x) in terms of clz (x). */ - if (unoptab == ffs_optab) - { - temp = expand_ffs (mode, op0, target); + if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode)) + { + temp = expand_ffs (int_mode, op0, target); if (temp) return temp; } /* Try implementing ctz (x) in terms of clz (x). */ - if (unoptab == ctz_optab) - { - temp = expand_ctz (mode, op0, target); + if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode)) + { + temp = expand_ctz (int_mode, op0, target); if (temp) return temp; } @@ -3216,15 +2947,16 @@ libfunc = optab_libfunc (unoptab, mode); if (libfunc) { - rtx insns; + rtx_insn *insns; rtx value; rtx eq_value; - enum machine_mode outmode = mode; + machine_mode outmode = mode; /* All of these functions return small values. Thus we choose to have them return something that isn't a double-word. */ if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab - || unoptab == popcount_optab || unoptab == parity_optab) + || unoptab == clrsb_optab || unoptab == popcount_optab + || unoptab == parity_optab) outmode = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node), optab_libfunc (unoptab, mode))); @@ -3234,17 +2966,24 @@ /* Pass 1 for NO_QUEUE so we don't lose any increments if the libcall is cse'd or moved. */ value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode, - 1, op0, mode); + op0, mode); insns = get_insns (); end_sequence (); target = gen_reg_rtx (outmode); - eq_value = gen_rtx_fmt_e (unoptab->code, mode, op0); - if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode)) - eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode); - else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode)) - eq_value = simplify_gen_unary (ZERO_EXTEND, outmode, eq_value, mode); - emit_libcall_block (insns, target, value, eq_value); + bool trapv = trapv_unoptab_p (unoptab); + if (trapv) + eq_value = NULL_RTX; + else + { + eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0); + if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode)) + eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode); + else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode)) + eq_value = simplify_gen_unary (ZERO_EXTEND, + outmode, eq_value, mode); + } + emit_libcall_block_1 (insns, target, value, eq_value, trapv); return target; } @@ -3253,35 +2992,58 @@ if (CLASS_HAS_WIDER_MODES_P (mclass)) { - for (wider_mode = GET_MODE_WIDER_MODE (mode); - wider_mode != VOIDmode; - wider_mode = GET_MODE_WIDER_MODE (wider_mode)) + FOR_EACH_WIDER_MODE (wider_mode, mode) { if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing || optab_libfunc (unoptab, wider_mode)) { rtx xop0 = op0; - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); /* For certain operations, we need not actually extend the narrow operand, as long as we will truncate the results to the same narrowness. */ - xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, (unoptab == neg_optab - || unoptab == one_cmpl_optab) + || unoptab == one_cmpl_optab + || unoptab == bswap_optab) && mclass == MODE_INT); temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, unsignedp); /* If we are generating clz using wider mode, adjust the - result. */ - if (unoptab == clz_optab && temp != 0) - temp = expand_binop (wider_mode, sub_optab, temp, - GEN_INT (GET_MODE_BITSIZE (wider_mode) - - GET_MODE_BITSIZE (mode)), - target, true, OPTAB_DIRECT); + result. Similarly for clrsb. */ + if ((unoptab == clz_optab || unoptab == clrsb_optab) + && temp != 0) + { + scalar_int_mode wider_int_mode + = as_a <scalar_int_mode> (wider_mode); + int_mode = as_a <scalar_int_mode> (mode); + temp = expand_binop + (wider_mode, sub_optab, temp, + gen_int_mode (GET_MODE_PRECISION (wider_int_mode) + - GET_MODE_PRECISION (int_mode), + wider_int_mode), + target, true, OPTAB_DIRECT); + } + + /* Likewise for bswap. */ + if (unoptab == bswap_optab && temp != 0) + { + scalar_int_mode wider_int_mode + = as_a <scalar_int_mode> (wider_mode); + int_mode = as_a <scalar_int_mode> (mode); + gcc_assert (GET_MODE_PRECISION (wider_int_mode) + == GET_MODE_BITSIZE (wider_int_mode) + && GET_MODE_PRECISION (int_mode) + == GET_MODE_BITSIZE (int_mode)); + + temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp, + GET_MODE_BITSIZE (wider_int_mode) + - GET_MODE_BITSIZE (int_mode), + NULL_RTX, true); + } if (temp) { @@ -3303,7 +3065,7 @@ /* One final attempt at implementing negation via subtraction, this time allowing widening of the operand. */ - if (unoptab->code == NEG && !HONOR_SIGNED_ZEROS (mode)) + if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode)) { rtx temp; temp = expand_binop (mode, @@ -3327,12 +3089,13 @@ */ rtx -expand_abs_nojump (enum machine_mode mode, rtx op0, rtx target, +expand_abs_nojump (machine_mode mode, rtx op0, rtx target, int result_unsignedp) { rtx temp; - if (! flag_trapv) + if (GET_MODE_CLASS (mode) != MODE_INT + || ! flag_trapv) result_unsignedp = 1; /* First try to do it with a special abs instruction. */ @@ -3342,9 +3105,10 @@ return temp; /* For floating point modes, try clearing the sign bit. */ - if (SCALAR_FLOAT_MODE_P (mode)) - { - temp = expand_absneg_bit (ABS, mode, op0, target); + scalar_float_mode float_mode; + if (is_a <scalar_float_mode> (mode, &float_mode)) + { + temp = expand_absneg_bit (ABS, float_mode, op0, target); if (temp) return temp; } @@ -3353,9 +3117,10 @@ if (optab_handler (smax_optab, mode) != CODE_FOR_nothing && !HONOR_SIGNED_ZEROS (mode)) { - rtx last = get_last_insn (); - - temp = expand_unop (mode, neg_optab, op0, NULL_RTX, 0); + rtx_insn *last = get_last_insn (); + + temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab, + op0, NULL_RTX, 0); if (temp != 0) temp = expand_binop (mode, smax_optab, op0, temp, target, 0, OPTAB_WIDEN); @@ -3370,18 +3135,20 @@ value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)), where W is the width of MODE. */ - if (GET_MODE_CLASS (mode) == MODE_INT + scalar_int_mode int_mode; + if (is_int_mode (mode, &int_mode) && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2) { - rtx extended = expand_shift (RSHIFT_EXPR, mode, op0, - size_int (GET_MODE_BITSIZE (mode) - 1), + rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0, + GET_MODE_PRECISION (int_mode) - 1, NULL_RTX, 0); - temp = expand_binop (mode, xor_optab, extended, op0, target, 0, + temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0, OPTAB_LIB_WIDEN); if (temp != 0) - temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab, + temp = expand_binop (int_mode, + result_unsignedp ? sub_optab : subv_optab, temp, extended, target, 0, OPTAB_LIB_WIDEN); if (temp != 0) @@ -3392,12 +3159,14 @@ } rtx -expand_abs (enum machine_mode mode, rtx op0, rtx target, +expand_abs (machine_mode mode, rtx op0, rtx target, int result_unsignedp, int safe) { - rtx temp, op1; - - if (! flag_trapv) + rtx temp; + rtx_code_label *op1; + + if (GET_MODE_CLASS (mode) != MODE_INT + || ! flag_trapv) result_unsignedp = 1; temp = expand_abs_nojump (mode, op0, target, result_unsignedp); @@ -3424,7 +3193,8 @@ NO_DEFER_POP; do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode, - NULL_RTX, NULL_RTX, op1, -1); + NULL_RTX, NULL, op1, + profile_probability::uninitialized ()); op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab, target, target, 0); @@ -3444,7 +3214,7 @@ different but can be deduced from MODE. */ rtx -expand_one_cmpl_abs_nojump (enum machine_mode mode, rtx op0, rtx target) +expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target) { rtx temp; @@ -3455,7 +3225,7 @@ /* If we have a MAX insn, we can do this as MAX (x, ~x). */ if (optab_handler (smax_optab, mode) != CODE_FOR_nothing) { - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0); if (temp != 0) @@ -3471,15 +3241,16 @@ /* If this machine has expensive jumps, we can do one's complement absolute value of X as (((signed) x >> (W-1)) ^ x). */ - if (GET_MODE_CLASS (mode) == MODE_INT + scalar_int_mode int_mode; + if (is_int_mode (mode, &int_mode) && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2) { - rtx extended = expand_shift (RSHIFT_EXPR, mode, op0, - size_int (GET_MODE_BITSIZE (mode) - 1), + rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0, + GET_MODE_PRECISION (int_mode) - 1, NULL_RTX, 0); - temp = expand_binop (mode, xor_optab, extended, op0, target, 0, + temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0, OPTAB_LIB_WIDEN); if (temp != 0) @@ -3495,33 +3266,31 @@ and not playing with subregs so much, will help the register allocator. */ static rtx -expand_copysign_absneg (enum machine_mode mode, rtx op0, rtx op1, rtx target, +expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target, int bitpos, bool op0_is_abs) { - enum machine_mode imode; - int icode; - rtx sign, label; + scalar_int_mode imode; + enum insn_code icode; + rtx sign; + rtx_code_label *label; if (target == op1) target = NULL_RTX; /* Check if the back end provides an insn that handles signbit for the argument's mode. */ - icode = (int) optab_handler (signbit_optab, mode); + icode = optab_handler (signbit_optab, mode); if (icode != CODE_FOR_nothing) { - imode = insn_data[icode].operand[0].mode; + imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode); sign = gen_reg_rtx (imode); emit_unop_insn (icode, sign, op1, UNKNOWN); } else { - double_int mask; - if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) { - imode = int_mode_for_mode (mode); - if (imode == BLKmode) + if (!int_mode_for_mode (mode).exists (&imode)) return NULL_RTX; op1 = gen_lowpart (imode, op1); } @@ -3538,10 +3307,9 @@ op1 = operand_subword_force (op1, word, mode); } - mask = double_int_setbit (double_int_zero, bitpos); - + wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); sign = expand_binop (imode, and_optab, op1, - immed_double_int_const (mask, imode), + immed_wide_int_const (mask, imode), NULL_RTX, 1, OPTAB_LIB_WIDEN); } @@ -3563,7 +3331,7 @@ label = gen_label_rtx (); emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label); - if (GET_CODE (op0) == CONST_DOUBLE) + if (CONST_DOUBLE_AS_FLOAT_P (op0)) op0 = simplify_unary_operation (NEG, mode, op0, mode); else op0 = expand_unop (mode, neg_optab, op0, target, 0); @@ -3581,18 +3349,17 @@ is true if op0 is known to have its sign bit clear. */ static rtx -expand_copysign_bit (enum machine_mode mode, rtx op0, rtx op1, rtx target, +expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target, int bitpos, bool op0_is_abs) { - enum machine_mode imode; - double_int mask; + scalar_int_mode imode; int word, nwords, i; - rtx temp, insns; + rtx temp; + rtx_insn *insns; if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) { - imode = int_mode_for_mode (mode); - if (imode == BLKmode) + if (!int_mode_for_mode (mode).exists (&imode)) return NULL_RTX; word = 0; nwords = 1; @@ -3609,9 +3376,12 @@ nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD; } - mask = double_int_setbit (double_int_zero, bitpos); - - if (target == 0 || target == op0 || target == op1) + wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); + + if (target == 0 + || target == op0 + || target == op1 + || (nwords > 1 && !valid_multiword_target_p (target))) target = gen_reg_rtx (mode); if (nwords > 1) @@ -3628,13 +3398,11 @@ if (!op0_is_abs) op0_piece = expand_binop (imode, and_optab, op0_piece, - immed_double_int_const (double_int_not (mask), - imode), + immed_wide_int_const (~mask, imode), NULL_RTX, 1, OPTAB_LIB_WIDEN); - op1 = expand_binop (imode, and_optab, operand_subword_force (op1, i, mode), - immed_double_int_const (mask, imode), + immed_wide_int_const (mask, imode), NULL_RTX, 1, OPTAB_LIB_WIDEN); temp = expand_binop (imode, ior_optab, op0_piece, op1, @@ -3654,14 +3422,13 @@ else { op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1), - immed_double_int_const (mask, imode), + immed_wide_int_const (mask, imode), NULL_RTX, 1, OPTAB_LIB_WIDEN); op0 = gen_lowpart (imode, op0); if (!op0_is_abs) op0 = expand_binop (imode, and_optab, op0, - immed_double_int_const (double_int_not (mask), - imode), + immed_wide_int_const (~mask, imode), NULL_RTX, 1, OPTAB_LIB_WIDEN); temp = expand_binop (imode, ior_optab, op0, op1, @@ -3679,12 +3446,12 @@ rtx expand_copysign (rtx op0, rtx op1, rtx target) { - enum machine_mode mode = GET_MODE (op0); + scalar_float_mode mode; const struct real_format *fmt; bool op0_is_abs; rtx temp; - gcc_assert (SCALAR_FLOAT_MODE_P (mode)); + mode = as_a <scalar_float_mode> (GET_MODE (op0)); gcc_assert (GET_MODE (op1) == mode); /* First try to do it with a special instruction. */ @@ -3698,7 +3465,7 @@ return NULL_RTX; op0_is_abs = false; - if (GET_CODE (op0) == CONST_DOUBLE) + if (CONST_DOUBLE_AS_FLOAT_P (op0)) { if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0))) op0 = simplify_unary_operation (ABS, mode, op0, mode); @@ -3706,7 +3473,7 @@ } if (fmt->signbit_ro >= 0 - && (GET_CODE (op0) == CONST_DOUBLE + && (CONST_DOUBLE_AS_FLOAT_P (op0) || (optab_handler (neg_optab, mode) != CODE_FOR_nothing && optab_handler (abs_optab, mode) != CODE_FOR_nothing))) { @@ -3731,37 +3498,26 @@ Return false if expansion failed. */ bool -maybe_emit_unop_insn (int icode, rtx target, rtx op0, enum rtx_code code) +maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0, + enum rtx_code code) { - rtx temp; - enum machine_mode mode0 = insn_data[icode].operand[1].mode; - rtx pat; - rtx last = get_last_insn (); - - temp = target; - - /* Now, if insn does not accept our operands, put them into pseudos. */ - - if (!insn_data[icode].operand[1].predicate (op0, mode0)) - op0 = copy_to_mode_reg (mode0, op0); - - if (!insn_data[icode].operand[0].predicate (temp, GET_MODE (temp))) - temp = gen_reg_rtx (GET_MODE (temp)); - - pat = GEN_FCN (icode) (temp, op0); + struct expand_operand ops[2]; + rtx_insn *pat; + + create_output_operand (&ops[0], target, GET_MODE (target)); + create_input_operand (&ops[1], op0, GET_MODE (op0)); + pat = maybe_gen_insn (icode, 2, ops); if (!pat) - { - delete_insns_since (last); - return false; - } - - if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX && code != UNKNOWN) - add_equal_note (pat, temp, code, op0, NULL_RTX); + return false; + + if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX + && code != UNKNOWN) + add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX); emit_insn (pat); - if (temp != target) - emit_move_insn (target, temp); + if (ops[0].value != target) + emit_move_insn (target, ops[0].value); return true; } /* Generate an instruction whose insn-code is INSN_CODE, @@ -3771,7 +3527,7 @@ the value that is stored into TARGET. */ void -emit_unop_insn (int icode, rtx target, rtx op0, enum rtx_code code) +emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code) { bool ok = maybe_emit_unop_insn (icode, target, op0, code); gcc_assert (ok); @@ -3779,7 +3535,8 @@ struct no_conflict_data { - rtx target, first, insn; + rtx target; + rtx_insn *first, *insn; bool must_stay; }; @@ -3830,11 +3587,12 @@ an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL note with an operand of EQUIV. */ -void -emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv) +static void +emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv, + bool equiv_may_trap) { rtx final_dest = target; - rtx next, last, insn; + rtx_insn *next, *last, *insn; /* If this is a reg with REG_USERVAR_P set, then it could possibly turn into a MEM later. Protect the libcall block from this change. */ @@ -3844,7 +3602,8 @@ /* If we're using non-call exceptions, a libcall corresponding to an operation that may trap may also trap. */ /* ??? See the comment in front of make_reg_eh_region_note. */ - if (cfun->can_throw_non_call_exceptions && may_trap_p (equiv)) + if (cfun->can_throw_non_call_exceptions + && (equiv_may_trap || may_trap_p (equiv))) { for (insn = insns; insn; insn = NEXT_INSN (insn)) if (CALL_P (insn)) @@ -3894,12 +3653,12 @@ if (! data.must_stay) { if (PREV_INSN (insn)) - NEXT_INSN (PREV_INSN (insn)) = next; + SET_NEXT_INSN (PREV_INSN (insn)) = next; else insns = next; if (next) - PREV_INSN (next) = PREV_INSN (insn); + SET_PREV_INSN (next) = PREV_INSN (insn); add_insn (insn); } @@ -3920,12 +3679,18 @@ } last = emit_move_insn (target, result); - if (optab_handler (mov_optab, GET_MODE (target)) != CODE_FOR_nothing) - set_unique_reg_note (last, REG_EQUAL, copy_rtx (equiv)); + if (equiv) + set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target); if (final_dest != target) emit_move_insn (final_dest, target); } + +void +emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv) +{ + emit_libcall_block_1 (insns, target, result, equiv, false); +} /* Nonzero if we can perform a comparison of mode MODE straightforwardly. PURPOSE describes how this comparison will be used. CODE is the rtx @@ -3936,28 +3701,28 @@ required to implement all (or any) of the unordered bcc operations. */ int -can_compare_p (enum rtx_code code, enum machine_mode mode, +can_compare_p (enum rtx_code code, machine_mode mode, enum can_compare_purpose purpose) { rtx test; test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx); do { - int icode; + enum insn_code icode; if (purpose == ccp_jump && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing - && insn_data[icode].operand[0].predicate (test, mode)) + && insn_operand_matches (icode, 0, test)) return 1; if (purpose == ccp_store_flag && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing - && insn_data[icode].operand[1].predicate (test, mode)) + && insn_operand_matches (icode, 1, test)) return 1; if (purpose == ccp_cmov && optab_handler (cmov_optab, mode) != CODE_FOR_nothing) return 1; - mode = GET_MODE_WIDER_MODE (mode); + mode = GET_MODE_WIDER_MODE (mode).else_void (); PUT_MODE (test, mode); } while (mode != VOIDmode); @@ -3966,13 +3731,17 @@ } /* This function is called when we are going to emit a compare instruction that - compares the values found in *PX and *PY, using the rtl operator COMPARISON. + compares the values found in X and Y, using the rtl operator COMPARISON. + + If they have mode BLKmode, then SIZE specifies the size of both operands. + + UNSIGNEDP nonzero says that the operands are unsigned; + this matters if they need to be widened (as given by METHODS). + + *PTEST is where the resulting comparison RTX is returned or NULL_RTX + if we failed to produce one. *PMODE is the mode of the inputs (in case they are const_int). - *PUNSIGNEDP nonzero says that the operands are unsigned; - this matters if they need to be widened (as given by METHODS). - - If they have mode BLKmode, then SIZE specifies the size of both operands. This function performs all the setup necessary so that the caller only has to emit a single comparison insn. This setup can involve doing a BLKmode @@ -3985,11 +3754,11 @@ static void prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size, int unsignedp, enum optab_methods methods, - rtx *ptest, enum machine_mode *pmode) + rtx *ptest, machine_mode *pmode) { - enum machine_mode mode = *pmode; + machine_mode mode = *pmode; rtx libfunc, test; - enum machine_mode cmp_mode; + machine_mode cmp_mode; enum mode_class mclass; /* The other methods are not needed. */ @@ -3998,16 +3767,16 @@ /* If we are optimizing, force expensive constants into a register. */ if (CONSTANT_P (x) && optimize - && (rtx_cost (x, COMPARE, optimize_insn_for_speed_p ()) + && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ()) > COSTS_N_INSNS (1))) x = force_reg (mode, x); if (CONSTANT_P (y) && optimize - && (rtx_cost (y, COMPARE, optimize_insn_for_speed_p ()) + && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ()) > COSTS_N_INSNS (1))) y = force_reg (mode, y); -#ifdef HAVE_cc0 +#if HAVE_cc0 /* Make sure if we have a canonical comparison. The RTL documentation states that canonical comparisons are required only for targets which have cc0. */ @@ -4024,10 +3793,8 @@ if (mode == BLKmode) { - enum machine_mode result_mode; + machine_mode result_mode; enum insn_code cmp_code; - tree length_type; - rtx libfunc; rtx result; rtx opalign = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT); @@ -4036,10 +3803,10 @@ /* Try to use a memory block compare insn - either cmpstr or cmpmem will do. */ - for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT); - cmp_mode != VOIDmode; - cmp_mode = GET_MODE_WIDER_MODE (cmp_mode)) + opt_scalar_int_mode cmp_mode_iter; + FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT) { + scalar_int_mode cmp_mode = cmp_mode_iter.require (); cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode); if (cmp_code == CODE_FOR_nothing) cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode); @@ -4049,10 +3816,10 @@ continue; /* Must make sure the size fits the insn's mode. */ - if ((CONST_INT_P (size) - && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode))) - || (GET_MODE_BITSIZE (GET_MODE (size)) - > GET_MODE_BITSIZE (cmp_mode))) + if (CONST_INT_P (size) + ? INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)) + : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size))) + > GET_MODE_BITSIZE (cmp_mode))) continue; result_mode = insn_data[cmp_code].operand[0].mode; @@ -4068,23 +3835,14 @@ if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN) goto fail; - /* Otherwise call a library function, memcmp. */ - libfunc = memcmp_libfunc; - length_type = sizetype; - result_mode = TYPE_MODE (integer_type_node); - cmp_mode = TYPE_MODE (length_type); - size = convert_to_mode (TYPE_MODE (length_type), size, - TYPE_UNSIGNED (length_type)); - - result = emit_library_call_value (libfunc, 0, LCT_PURE, - result_mode, 3, - XEXP (x, 0), Pmode, - XEXP (y, 0), Pmode, - size, cmp_mode); - - *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx); - *pmode = result_mode; - return; + /* Otherwise call a library function. */ + result = emit_block_comp_via_libcall (XEXP (x, 0), XEXP (y, 0), size); + + x = result; + y = const0_rtx; + mode = TYPE_MODE (integer_type_node); + methods = OPTAB_LIB_WIDEN; + unsignedp = false; } /* Don't allow operands to the compare to trap, as that can put the @@ -4092,36 +3850,36 @@ if (cfun->can_throw_non_call_exceptions) { if (may_trap_p (x)) - x = force_reg (mode, x); + x = copy_to_reg (x); if (may_trap_p (y)) - y = force_reg (mode, y); + y = copy_to_reg (y); } if (GET_MODE_CLASS (mode) == MODE_CC) { - gcc_assert (can_compare_p (comparison, CCmode, ccp_jump)); - *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, x, y); + enum insn_code icode = optab_handler (cbranch_optab, CCmode); + test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y); + gcc_assert (icode != CODE_FOR_nothing + && insn_operand_matches (icode, 0, test)); + *ptest = test; return; } mclass = GET_MODE_CLASS (mode); test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y); - cmp_mode = mode; - do - { + FOR_EACH_MODE_FROM (cmp_mode, mode) + { enum insn_code icode; icode = optab_handler (cbranch_optab, cmp_mode); if (icode != CODE_FOR_nothing - && insn_data[icode].operand[0].predicate (test, VOIDmode)) + && insn_operand_matches (icode, 0, test)) { - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp); rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp); if (op0 && op1 - && insn_data[icode].operand[1].predicate - (op0, insn_data[icode].operand[1].mode) - && insn_data[icode].operand[2].predicate - (op1, insn_data[icode].operand[2].mode)) + && insn_operand_matches (icode, 1, op0) + && insn_operand_matches (icode, 2, op1)) { XEXP (test, 0) = op0; XEXP (test, 1) = op1; @@ -4134,9 +3892,7 @@ if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass)) break; - cmp_mode = GET_MODE_WIDER_MODE (cmp_mode); - } - while (cmp_mode != VOIDmode); + } if (methods != OPTAB_LIB_WIDEN) goto fail; @@ -4144,6 +3900,7 @@ if (!SCALAR_FLOAT_MODE_P (mode)) { rtx result; + machine_mode ret_mode; /* Handle a libcall just for the mode we are using. */ libfunc = optab_libfunc (cmp_optab, mode); @@ -4158,9 +3915,9 @@ libfunc = ulibfunc; } + ret_mode = targetm.libgcc_cmp_return_mode (); result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, - targetm.libgcc_cmp_return_mode (), - 2, x, mode, y, mode); + ret_mode, x, mode, y, mode); /* There are two kinds of comparison routines. Biased routines return 0/1/2, and unbiased routines return -1/0/1. Other parts @@ -4169,19 +3926,21 @@ result against 1 in the biased case, and zero in the unbiased case. For unsigned comparisons always compare against 1 after biasing the unbiased result by adding 1. This gives us a way to - represent LTU. */ + represent LTU. + The comparisons in the fixed-point helper library are always + biased. */ x = result; y = const1_rtx; - if (!TARGET_LIB_INT_CMP_BIASED) + if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode)) { if (unsignedp) - x = plus_constant (result, 1); + x = plus_constant (ret_mode, result, 1); else y = const0_rtx; } - *pmode = word_mode; + *pmode = ret_mode; prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods, ptest, pmode); } @@ -4200,18 +3959,20 @@ that it is accepted by the operand predicate. Return the new value. */ rtx -prepare_operand (int icode, rtx x, int opnum, enum machine_mode mode, - enum machine_mode wider_mode, int unsignedp) +prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode, + machine_mode wider_mode, int unsignedp) { if (mode != wider_mode) x = convert_modes (wider_mode, mode, x, unsignedp); - if (!insn_data[icode].operand[opnum].predicate - (x, insn_data[icode].operand[opnum].mode)) - { + if (!insn_operand_matches (icode, opnum, x)) + { + machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode; if (reload_completed) return NULL_RTX; - x = copy_to_mode_reg (insn_data[icode].operand[opnum].mode, x); + if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode) + return NULL_RTX; + x = copy_to_mode_reg (op_mode, x); } return x; @@ -4221,19 +3982,29 @@ we can do the branch. */ static void -emit_cmp_and_jump_insn_1 (rtx test, enum machine_mode mode, rtx label) +emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, + profile_probability prob) { - enum machine_mode optab_mode; + machine_mode optab_mode; enum mode_class mclass; enum insn_code icode; + rtx_insn *insn; mclass = GET_MODE_CLASS (mode); optab_mode = (mclass == MODE_CC) ? CCmode : mode; icode = optab_handler (cbranch_optab, optab_mode); gcc_assert (icode != CODE_FOR_nothing); - gcc_assert (insn_data[icode].operand[0].predicate (test, VOIDmode)); - emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0), XEXP (test, 1), label)); + gcc_assert (insn_operand_matches (icode, 0, test)); + insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0), + XEXP (test, 1), label)); + if (prob.initialized_p () + && profile_status_for_fn (cfun) != PROFILE_ABSENT + && insn + && JUMP_P (insn) + && any_condjump_p (insn) + && !find_reg_note (insn, REG_BR_PROB, 0)) + add_reg_br_prob_note (insn, prob); } /* Generate code to compare X with Y so that the condition codes are @@ -4251,11 +4022,14 @@ COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). It will be potentially converted into an unsigned variant based on - UNSIGNEDP to select a proper jump instruction. */ + UNSIGNEDP to select a proper jump instruction. + + PROB is the probability of jumping to LABEL. */ void emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size, - enum machine_mode mode, int unsignedp, rtx label) + machine_mode mode, int unsignedp, rtx label, + profile_probability prob) { rtx op0 = x, op1 = y; rtx test; @@ -4279,7 +4053,7 @@ prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN, &test, &mode); - emit_cmp_and_jump_insn_1 (test, mode, label); + emit_cmp_and_jump_insn_1 (test, mode, label, prob); } @@ -4288,37 +4062,35 @@ static void prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison, - rtx *ptest, enum machine_mode *pmode) + rtx *ptest, machine_mode *pmode) { enum rtx_code swapped = swap_condition (comparison); enum rtx_code reversed = reverse_condition_maybe_unordered (comparison); - enum machine_mode orig_mode = GET_MODE (x); - enum machine_mode mode, cmp_mode; + machine_mode orig_mode = GET_MODE (x); + machine_mode mode; rtx true_rtx, false_rtx; - rtx value, target, insns, equiv; + rtx value, target, equiv; + rtx_insn *insns; rtx libfunc = 0; bool reversed_p = false; - cmp_mode = targetm.libgcc_cmp_return_mode (); - - for (mode = orig_mode; - mode != VOIDmode; - mode = GET_MODE_WIDER_MODE (mode)) - { - if (code_to_optab[comparison] - && (libfunc = optab_libfunc (code_to_optab[comparison], mode))) + scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode (); + + FOR_EACH_MODE_FROM (mode, orig_mode) + { + if (code_to_optab (comparison) + && (libfunc = optab_libfunc (code_to_optab (comparison), mode))) break; - if (code_to_optab[swapped] - && (libfunc = optab_libfunc (code_to_optab[swapped], mode))) + if (code_to_optab (swapped) + && (libfunc = optab_libfunc (code_to_optab (swapped), mode))) { - rtx tmp; - tmp = x; x = y; y = tmp; + std::swap (x, y); comparison = swapped; break; } - if (code_to_optab[reversed] - && (libfunc = optab_libfunc (code_to_optab[reversed], mode))) + if (code_to_optab (reversed) + && (libfunc = optab_libfunc (code_to_optab (reversed), mode))) { comparison = reversed; reversed_p = true; @@ -4399,7 +4171,7 @@ start_sequence (); value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, - cmp_mode, 2, x, mode, y, mode); + cmp_mode, x, mode, y, mode); insns = get_insns (); end_sequence (); @@ -4421,15 +4193,17 @@ void emit_indirect_jump (rtx loc) { - if (!insn_data[(int) CODE_FOR_indirect_jump].operand[0].predicate - (loc, Pmode)) - loc = copy_to_mode_reg (Pmode, loc); - - emit_jump_insn (gen_indirect_jump (loc)); - emit_barrier (); + if (!targetm.have_indirect_jump ()) + sorry ("indirect jumps are not available on this target"); + else + { + struct expand_operand ops[1]; + create_address_operand (&ops[0], loc); + expand_jump_insn (targetm.code_for_indirect_jump, 1, ops); + emit_barrier (); + } } -#ifdef HAVE_conditional_move /* Emit a conditional move instruction if the machine supports one for that condition and machine mode. @@ -4447,21 +4221,31 @@ rtx emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1, - enum machine_mode cmode, rtx op2, rtx op3, - enum machine_mode mode, int unsignedp) + machine_mode cmode, rtx op2, rtx op3, + machine_mode mode, int unsignedp) { - rtx tem, subtarget, comparison, insn; + rtx comparison; + rtx_insn *last; enum insn_code icode; enum rtx_code reversed; + /* If the two source operands are identical, that's just a move. */ + + if (rtx_equal_p (op2, op3)) + { + if (!target) + target = gen_reg_rtx (mode); + + emit_move_insn (target, op3); + return target; + } + /* If one operand is constant, make it the second one. Only do this if the other operand is not constant as well. */ if (swap_commutative_operands_p (op0, op1)) { - tem = op0; - op0 = op1; - op1 = tem; + std::swap (op0, op1); code = swap_condition (code); } @@ -4476,14 +4260,15 @@ if (cmode == VOIDmode) cmode = GET_MODE (op0); + enum rtx_code orig_code = code; + bool swapped = false; if (swap_commutative_operands_p (op2, op3) && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL)) != UNKNOWN)) { - tem = op2; - op2 = op3; - op3 = tem; + std::swap (op2, op3); code = reversed; + swapped = true; } if (mode == VOIDmode) @@ -4492,84 +4277,114 @@ icode = direct_optab_handler (movcc_optab, mode); if (icode == CODE_FOR_nothing) - return 0; + return NULL_RTX; if (!target) target = gen_reg_rtx (mode); - subtarget = target; - - /* If the insn doesn't accept these operands, put them in pseudos. */ - - if (!insn_data[icode].operand[0].predicate - (subtarget, insn_data[icode].operand[0].mode)) - subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode); - - if (!insn_data[icode].operand[2].predicate - (op2, insn_data[icode].operand[2].mode)) - op2 = copy_to_mode_reg (insn_data[icode].operand[2].mode, op2); - - if (!insn_data[icode].operand[3].predicate - (op3, insn_data[icode].operand[3].mode)) - op3 = copy_to_mode_reg (insn_data[icode].operand[3].mode, op3); - - /* Everything should now be in the suitable form. */ - - code = unsignedp ? unsigned_condition (code) : code; - comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1); - - /* We can get const0_rtx or const_true_rtx in some circumstances. Just - return NULL and let the caller figure out how best to deal with this - situation. */ - if (!COMPARISON_P (comparison)) - return NULL_RTX; - - do_pending_stack_adjust (); - start_sequence (); - prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), - GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN, - &comparison, &cmode); - if (!comparison) - insn = NULL_RTX; + for (int pass = 0; ; pass++) + { + code = unsignedp ? unsigned_condition (code) : code; + comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1); + + /* We can get const0_rtx or const_true_rtx in some circumstances. Just + punt and let the caller figure out how best to deal with this + situation. */ + if (COMPARISON_P (comparison)) + { + saved_pending_stack_adjust save; + save_pending_stack_adjust (&save); + last = get_last_insn (); + do_pending_stack_adjust (); + prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), + GET_CODE (comparison), NULL_RTX, unsignedp, + OPTAB_WIDEN, &comparison, &cmode); + if (comparison) + { + struct expand_operand ops[4]; + + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], comparison); + create_input_operand (&ops[2], op2, mode); + create_input_operand (&ops[3], op3, mode); + if (maybe_expand_insn (icode, 4, ops)) + { + if (ops[0].value != target) + convert_move (target, ops[0].value, false); + return target; + } + } + delete_insns_since (last); + restore_pending_stack_adjust (&save); + } + + if (pass == 1) + return NULL_RTX; + + /* If the preferred op2/op3 order is not usable, retry with other + operand order, perhaps it will expand successfully. */ + if (swapped) + code = orig_code; + else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1, + NULL)) + != UNKNOWN) + code = reversed; + else + return NULL_RTX; + std::swap (op2, op3); + } +} + + +/* Emit a conditional negate or bitwise complement using the + negcc or notcc optabs if available. Return NULL_RTX if such operations + are not available. Otherwise return the RTX holding the result. + TARGET is the desired destination of the result. COMP is the comparison + on which to negate. If COND is true move into TARGET the negation + or bitwise complement of OP1. Otherwise move OP2 into TARGET. + CODE is either NEG or NOT. MODE is the machine mode in which the + operation is performed. */ + +rtx +emit_conditional_neg_or_complement (rtx target, rtx_code code, + machine_mode mode, rtx cond, rtx op1, + rtx op2) +{ + optab op = unknown_optab; + if (code == NEG) + op = negcc_optab; + else if (code == NOT) + op = notcc_optab; else - insn = GEN_FCN (icode) (subtarget, comparison, op2, op3); - - /* If that failed, then give up. */ - if (insn == 0) - { - end_sequence (); - return 0; - } - - emit_insn (insn); - insn = get_insns (); - end_sequence (); - emit_insn (insn); - if (subtarget != target) - convert_move (target, subtarget, 0); - - return target; + gcc_unreachable (); + + insn_code icode = direct_optab_handler (op, mode); + + if (icode == CODE_FOR_nothing) + return NULL_RTX; + + if (!target) + target = gen_reg_rtx (mode); + + rtx_insn *last = get_last_insn (); + struct expand_operand ops[4]; + + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], cond); + create_input_operand (&ops[2], op1, mode); + create_input_operand (&ops[3], op2, mode); + + if (maybe_expand_insn (icode, 4, ops)) + { + if (ops[0].value != target) + convert_move (target, ops[0].value, false); + + return target; + } + delete_insns_since (last); + return NULL_RTX; } -/* Return nonzero if a conditional move of mode MODE is supported. - - This function is for combine so it can tell whether an insn that looks - like a conditional move is actually supported by the hardware. If we - guess wrong we lose a bit on optimization, but that's it. */ -/* ??? sparc64 supports conditionally moving integers values based on fp - comparisons, and vice versa. How do we handle them? */ - -int -can_conditionally_move_p (enum machine_mode mode) -{ - if (direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing) - return 1; - - return 0; -} - -#endif /* HAVE_conditional_move */ - /* Emit a conditional addition instruction if the machine supports one for that condition and machine mode. @@ -4577,7 +4392,7 @@ the mode to use should they be constants. If it is VOIDmode, they cannot both be constants. - OP2 should be stored in TARGET if the comparison is true, otherwise OP2+OP3 + OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3 should be stored there. MODE is the mode to use should they be constants. If it is VOIDmode, they cannot both be constants. @@ -4586,21 +4401,19 @@ rtx emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1, - enum machine_mode cmode, rtx op2, rtx op3, - enum machine_mode mode, int unsignedp) + machine_mode cmode, rtx op2, rtx op3, + machine_mode mode, int unsignedp) { - rtx tem, subtarget, comparison, insn; + rtx comparison; + rtx_insn *last; enum insn_code icode; - enum rtx_code reversed; /* If one operand is constant, make it the second one. Only do this if the other operand is not constant as well. */ if (swap_commutative_operands_p (op0, op1)) { - tem = op0; - op0 = op1; - op1 = tem; + std::swap (op0, op1); code = swap_condition (code); } @@ -4615,16 +4428,6 @@ if (cmode == VOIDmode) cmode = GET_MODE (op0); - if (swap_commutative_operands_p (op2, op3) - && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL)) - != UNKNOWN)) - { - tem = op2; - op2 = op3; - op3 = tem; - code = reversed; - } - if (mode == VOIDmode) mode = GET_MODE (op2); @@ -4636,24 +4439,6 @@ if (!target) target = gen_reg_rtx (mode); - /* If the insn doesn't accept these operands, put them in pseudos. */ - - if (!insn_data[icode].operand[0].predicate - (target, insn_data[icode].operand[0].mode)) - subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode); - else - subtarget = target; - - if (!insn_data[icode].operand[2].predicate - (op2, insn_data[icode].operand[2].mode)) - op2 = copy_to_mode_reg (insn_data[icode].operand[2].mode, op2); - - if (!insn_data[icode].operand[3].predicate - (op3, insn_data[icode].operand[3].mode)) - op3 = copy_to_mode_reg (insn_data[icode].operand[3].mode, op3); - - /* Everything should now be in the suitable form. */ - code = unsignedp ? unsigned_condition (code) : code; comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1); @@ -4664,30 +4449,27 @@ return NULL_RTX; do_pending_stack_adjust (); - start_sequence (); + last = get_last_insn (); prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN, &comparison, &cmode); - if (!comparison) - insn = NULL_RTX; - else - insn = GEN_FCN (icode) (subtarget, comparison, op2, op3); - - /* If that failed, then give up. */ - if (insn == 0) - { - end_sequence (); - return 0; - } - - emit_insn (insn); - insn = get_insns (); - end_sequence (); - emit_insn (insn); - if (subtarget != target) - convert_move (target, subtarget, 0); - - return target; + if (comparison) + { + struct expand_operand ops[4]; + + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], comparison); + create_input_operand (&ops[2], op2, mode); + create_input_operand (&ops[3], op3, mode); + if (maybe_expand_insn (icode, 4, ops)) + { + if (ops[0].value != target) + convert_move (target, ops[0].value, false); + return target; + } + } + delete_insns_since (last); + return NULL_RTX; } /* These functions attempt to generate an insn body, rather than @@ -4696,17 +4478,14 @@ /* Generate and return an insn body to add Y to X. */ -rtx +rtx_insn * gen_add2_insn (rtx x, rtx y) { - int icode = (int) optab_handler (add_optab, GET_MODE (x)); - - gcc_assert (insn_data[icode].operand[0].predicate - (x, insn_data[icode].operand[0].mode)); - gcc_assert (insn_data[icode].operand[1].predicate - (x, insn_data[icode].operand[1].mode)); - gcc_assert (insn_data[icode].operand[2].predicate - (y, insn_data[icode].operand[2].mode)); + enum insn_code icode = optab_handler (add_optab, GET_MODE (x)); + + gcc_assert (insn_operand_matches (icode, 0, x)); + gcc_assert (insn_operand_matches (icode, 1, x)); + gcc_assert (insn_operand_matches (icode, 2, y)); return GEN_FCN (icode) (x, x, y); } @@ -4714,19 +4493,16 @@ /* Generate and return an insn body to add r1 and c, storing the result in r0. */ -rtx +rtx_insn * gen_add3_insn (rtx r0, rtx r1, rtx c) { - int icode = (int) optab_handler (add_optab, GET_MODE (r0)); + enum insn_code icode = optab_handler (add_optab, GET_MODE (r0)); if (icode == CODE_FOR_nothing - || !(insn_data[icode].operand[0].predicate - (r0, insn_data[icode].operand[0].mode)) - || !(insn_data[icode].operand[1].predicate - (r1, insn_data[icode].operand[1].mode)) - || !(insn_data[icode].operand[2].predicate - (c, insn_data[icode].operand[2].mode))) - return NULL_RTX; + || !insn_operand_matches (icode, 0, r0) + || !insn_operand_matches (icode, 1, r1) + || !insn_operand_matches (icode, 2, c)) + return NULL; return GEN_FCN (icode) (r0, r1, c); } @@ -4734,21 +4510,55 @@ int have_add2_insn (rtx x, rtx y) { - int icode; + enum insn_code icode; gcc_assert (GET_MODE (x) != VOIDmode); - icode = (int) optab_handler (add_optab, GET_MODE (x)); + icode = optab_handler (add_optab, GET_MODE (x)); if (icode == CODE_FOR_nothing) return 0; - if (!(insn_data[icode].operand[0].predicate - (x, insn_data[icode].operand[0].mode)) - || !(insn_data[icode].operand[1].predicate - (x, insn_data[icode].operand[1].mode)) - || !(insn_data[icode].operand[2].predicate - (y, insn_data[icode].operand[2].mode))) + if (!insn_operand_matches (icode, 0, x) + || !insn_operand_matches (icode, 1, x) + || !insn_operand_matches (icode, 2, y)) + return 0; + + return 1; +} + +/* Generate and return an insn body to add Y to X. */ + +rtx_insn * +gen_addptr3_insn (rtx x, rtx y, rtx z) +{ + enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x)); + + gcc_assert (insn_operand_matches (icode, 0, x)); + gcc_assert (insn_operand_matches (icode, 1, y)); + gcc_assert (insn_operand_matches (icode, 2, z)); + + return GEN_FCN (icode) (x, y, z); +} + +/* Return true if the target implements an addptr pattern and X, Y, + and Z are valid for the pattern predicates. */ + +int +have_addptr3_insn (rtx x, rtx y, rtx z) +{ + enum insn_code icode; + + gcc_assert (GET_MODE (x) != VOIDmode); + + icode = optab_handler (addptr3_optab, GET_MODE (x)); + + if (icode == CODE_FOR_nothing) + return 0; + + if (!insn_operand_matches (icode, 0, x) + || !insn_operand_matches (icode, 1, y) + || !insn_operand_matches (icode, 2, z)) return 0; return 1; @@ -4756,17 +4566,14 @@ /* Generate and return an insn body to subtract Y from X. */ -rtx +rtx_insn * gen_sub2_insn (rtx x, rtx y) { - int icode = (int) optab_handler (sub_optab, GET_MODE (x)); - - gcc_assert (insn_data[icode].operand[0].predicate - (x, insn_data[icode].operand[0].mode)); - gcc_assert (insn_data[icode].operand[1].predicate - (x, insn_data[icode].operand[1].mode)); - gcc_assert (insn_data[icode].operand[2].predicate - (y, insn_data[icode].operand[2].mode)); + enum insn_code icode = optab_handler (sub_optab, GET_MODE (x)); + + gcc_assert (insn_operand_matches (icode, 0, x)); + gcc_assert (insn_operand_matches (icode, 1, x)); + gcc_assert (insn_operand_matches (icode, 2, y)); return GEN_FCN (icode) (x, x, y); } @@ -4774,19 +4581,16 @@ /* Generate and return an insn body to subtract r1 and c, storing the result in r0. */ -rtx +rtx_insn * gen_sub3_insn (rtx r0, rtx r1, rtx c) { - int icode = (int) optab_handler (sub_optab, GET_MODE (r0)); + enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0)); if (icode == CODE_FOR_nothing - || !(insn_data[icode].operand[0].predicate - (r0, insn_data[icode].operand[0].mode)) - || !(insn_data[icode].operand[1].predicate - (r1, insn_data[icode].operand[1].mode)) - || !(insn_data[icode].operand[2].predicate - (c, insn_data[icode].operand[2].mode))) - return NULL_RTX; + || !insn_operand_matches (icode, 0, r0) + || !insn_operand_matches (icode, 1, r1) + || !insn_operand_matches (icode, 2, c)) + return NULL; return GEN_FCN (icode) (r0, r1, c); } @@ -4794,120 +4598,34 @@ int have_sub2_insn (rtx x, rtx y) { - int icode; + enum insn_code icode; gcc_assert (GET_MODE (x) != VOIDmode); - icode = (int) optab_handler (sub_optab, GET_MODE (x)); + icode = optab_handler (sub_optab, GET_MODE (x)); if (icode == CODE_FOR_nothing) return 0; - if (!(insn_data[icode].operand[0].predicate - (x, insn_data[icode].operand[0].mode)) - || !(insn_data[icode].operand[1].predicate - (x, insn_data[icode].operand[1].mode)) - || !(insn_data[icode].operand[2].predicate - (y, insn_data[icode].operand[2].mode))) + if (!insn_operand_matches (icode, 0, x) + || !insn_operand_matches (icode, 1, x) + || !insn_operand_matches (icode, 2, y)) return 0; return 1; } - -/* Generate the body of an instruction to copy Y into X. - It may be a list of insns, if one insn isn't enough. */ - -rtx -gen_move_insn (rtx x, rtx y) -{ - rtx seq; - - start_sequence (); - emit_move_insn_1 (x, y); - seq = get_insns (); - end_sequence (); - return seq; -} -/* Return the insn code used to extend FROM_MODE to TO_MODE. - UNSIGNEDP specifies zero-extension instead of sign-extension. If - no such operation exists, CODE_FOR_nothing will be returned. */ - -enum insn_code -can_extend_p (enum machine_mode to_mode, enum machine_mode from_mode, - int unsignedp) -{ - convert_optab tab; -#ifdef HAVE_ptr_extend - if (unsignedp < 0) - return CODE_FOR_ptr_extend; -#endif - - tab = unsignedp ? zext_optab : sext_optab; - return convert_optab_handler (tab, to_mode, from_mode); -} - /* Generate the body of an insn to extend Y (with mode MFROM) into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */ -rtx -gen_extend_insn (rtx x, rtx y, enum machine_mode mto, - enum machine_mode mfrom, int unsignedp) +rtx_insn * +gen_extend_insn (rtx x, rtx y, machine_mode mto, + machine_mode mfrom, int unsignedp) { enum insn_code icode = can_extend_p (mto, mfrom, unsignedp); return GEN_FCN (icode) (x, y); } -/* can_fix_p and can_float_p say whether the target machine - can directly convert a given fixed point type to - a given floating point type, or vice versa. - The returned value is the CODE_FOR_... value to use, - or CODE_FOR_nothing if these modes cannot be directly converted. - - *TRUNCP_PTR is set to 1 if it is necessary to output - an explicit FTRUNC insn before the fix insn; otherwise 0. */ - -static enum insn_code -can_fix_p (enum machine_mode fixmode, enum machine_mode fltmode, - int unsignedp, int *truncp_ptr) -{ - convert_optab tab; - enum insn_code icode; - - tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab; - icode = convert_optab_handler (tab, fixmode, fltmode); - if (icode != CODE_FOR_nothing) - { - *truncp_ptr = 0; - return icode; - } - - /* FIXME: This requires a port to define both FIX and FTRUNC pattern - for this to work. We need to rework the fix* and ftrunc* patterns - and documentation. */ - tab = unsignedp ? ufix_optab : sfix_optab; - icode = convert_optab_handler (tab, fixmode, fltmode); - if (icode != CODE_FOR_nothing - && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing) - { - *truncp_ptr = 1; - return icode; - } - - *truncp_ptr = 0; - return CODE_FOR_nothing; -} - -static enum insn_code -can_float_p (enum machine_mode fltmode, enum machine_mode fixmode, - int unsignedp) -{ - convert_optab tab; - - tab = unsignedp ? ufloat_optab : sfloat_optab; - return convert_optab_handler (tab, fltmode, fixmode); -} - /* Generate code to convert FROM to floating point and store in TO. FROM must be fixed point and not VOIDmode. UNSIGNEDP nonzero means regard FROM as unsigned. @@ -4919,7 +4637,8 @@ { enum insn_code icode; rtx target = to; - enum machine_mode fmode, imode; + scalar_mode from_mode, to_mode; + machine_mode fmode, imode; bool can_do_signed = false; /* Crash now, because we won't be able to decide which mode to use. */ @@ -4930,15 +4649,14 @@ wider mode. If the integer mode is wider than the mode of FROM, we can do the conversion signed even if the input is unsigned. */ - for (fmode = GET_MODE (to); fmode != VOIDmode; - fmode = GET_MODE_WIDER_MODE (fmode)) - for (imode = GET_MODE (from); imode != VOIDmode; - imode = GET_MODE_WIDER_MODE (imode)) + FOR_EACH_MODE_FROM (fmode, GET_MODE (to)) + FOR_EACH_MODE_FROM (imode, GET_MODE (from)) { int doing_unsigned = unsignedp; if (fmode != GET_MODE (to) - && significand_size (fmode) < GET_MODE_BITSIZE (GET_MODE (from))) + && (significand_size (fmode) + < GET_MODE_UNIT_PRECISION (GET_MODE (from)))) continue; icode = can_float_p (fmode, imode, unsignedp); @@ -4970,9 +4688,13 @@ /* Unsigned integer, and no way to convert directly. Convert as signed, then unconditionally adjust the result. */ - if (unsignedp && can_do_signed) - { - rtx label = gen_label_rtx (); + if (unsignedp + && can_do_signed + && is_a <scalar_mode> (GET_MODE (to), &to_mode) + && is_a <scalar_mode> (GET_MODE (from), &from_mode)) + { + opt_scalar_mode fmode_iter; + rtx_code_label *label = gen_label_rtx (); rtx temp; REAL_VALUE_TYPE offset; @@ -4980,23 +4702,25 @@ least as wide as the target. Using FMODE will avoid rounding woes with unsigned values greater than the signed maximum value. */ - for (fmode = GET_MODE (to); fmode != VOIDmode; - fmode = GET_MODE_WIDER_MODE (fmode)) - if (GET_MODE_BITSIZE (GET_MODE (from)) < GET_MODE_BITSIZE (fmode) - && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing) - break; - - if (fmode == VOIDmode) + FOR_EACH_MODE_FROM (fmode_iter, to_mode) + { + scalar_mode fmode = fmode_iter.require (); + if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode) + && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing) + break; + } + + if (!fmode_iter.exists (&fmode)) { /* There is no such mode. Pretend the target is wide enough. */ - fmode = GET_MODE (to); + fmode = to_mode; /* Avoid double-rounding when TO is narrower than FROM. */ if ((significand_size (fmode) + 1) - < GET_MODE_BITSIZE (GET_MODE (from))) + < GET_MODE_PRECISION (from_mode)) { rtx temp1; - rtx neglabel = gen_label_rtx (); + rtx_code_label *neglabel = gen_label_rtx (); /* Don't use TARGET if it isn't a register, is a hard register, or is the wrong mode. */ @@ -5005,7 +4729,7 @@ || GET_MODE (target) != fmode) target = gen_reg_rtx (fmode); - imode = GET_MODE (from); + imode = from_mode; do_pending_stack_adjust (); /* Test whether the sign bit is set. */ @@ -5014,7 +4738,7 @@ /* The sign bit is not set. Convert as signed. */ expand_float (target, from, 0); - emit_jump_insn (gen_jump (label)); + emit_jump_insn (targetm.gen_jump (label)); emit_barrier (); /* The sign bit is set. @@ -5025,8 +4749,7 @@ emit_label (neglabel); temp = expand_binop (imode, and_optab, from, const1_rtx, NULL_RTX, 1, OPTAB_LIB_WIDEN); - temp1 = expand_shift (RSHIFT_EXPR, imode, from, integer_one_node, - NULL_RTX, 1); + temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1); temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1, OPTAB_LIB_WIDEN); expand_float (target, temp, 0); @@ -5046,7 +4769,7 @@ /* If we are about to do some arithmetic to correct for an unsigned operand, do it in a pseudo-register. */ - if (GET_MODE (to) != fmode + if (to_mode != fmode || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER) target = gen_reg_rtx (fmode); @@ -5057,13 +4780,13 @@ correct its value by 2**bitwidth. */ do_pending_stack_adjust (); - emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from), + emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode, 0, label); - real_2expN (&offset, GET_MODE_BITSIZE (GET_MODE (from)), fmode); + real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode); temp = expand_binop (fmode, add_optab, target, - CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode), + const_double_from_real_value (offset, fmode), target, 0, OPTAB_LIB_WIDEN); if (temp != target) emit_move_insn (target, temp); @@ -5076,11 +4799,11 @@ /* No hardware instruction available; call a library routine. */ { rtx libfunc; - rtx insns; + rtx_insn *insns; rtx value; convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab; - if (GET_MODE_SIZE (GET_MODE (from)) < GET_MODE_SIZE (SImode)) + if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_PRECISION (SImode)) from = convert_to_mode (SImode, from, unsignedp); libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from)); @@ -5089,8 +4812,7 @@ start_sequence (); value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, - GET_MODE (to), 1, from, - GET_MODE (from)); + GET_MODE (to), from, GET_MODE (from)); insns = get_insns (); end_sequence (); @@ -5121,18 +4843,17 @@ { enum insn_code icode; rtx target = to; - enum machine_mode fmode, imode; - int must_trunc = 0; + machine_mode fmode, imode; + opt_scalar_mode fmode_iter; + bool must_trunc = false; /* We first try to find a pair of modes, one real and one integer, at least as wide as FROM and TO, respectively, in which we can open-code this conversion. If the integer mode is wider than the mode of TO, we can do the conversion either signed or unsigned. */ - for (fmode = GET_MODE (from); fmode != VOIDmode; - fmode = GET_MODE_WIDER_MODE (fmode)) - for (imode = GET_MODE (to); imode != VOIDmode; - imode = GET_MODE_WIDER_MODE (imode)) + FOR_EACH_MODE_FROM (fmode, GET_MODE (from)) + FOR_EACH_MODE_FROM (imode, GET_MODE (to)) { int doing_unsigned = unsignedp; @@ -5142,7 +4863,7 @@ if (icode != CODE_FOR_nothing) { - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); if (fmode != GET_MODE (from)) from = convert_to_mode (fmode, from, 0); @@ -5191,73 +4912,80 @@ 2^63. The subtraction of 2^63 should not generate any rounding as it simply clears out that bit. The rest is trivial. */ - if (unsignedp && GET_MODE_BITSIZE (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT) - for (fmode = GET_MODE (from); fmode != VOIDmode; - fmode = GET_MODE_WIDER_MODE (fmode)) - if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc) - && (!DECIMAL_FLOAT_MODE_P (fmode) - || GET_MODE_BITSIZE (fmode) > GET_MODE_BITSIZE (GET_MODE (to)))) - { - int bitsize; - REAL_VALUE_TYPE offset; - rtx limit, lab1, lab2, insn; - - bitsize = GET_MODE_BITSIZE (GET_MODE (to)); - real_2expN (&offset, bitsize - 1, fmode); - limit = CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode); - lab1 = gen_label_rtx (); - lab2 = gen_label_rtx (); - - if (fmode != GET_MODE (from)) - from = convert_to_mode (fmode, from, 0); - - /* See if we need to do the subtraction. */ - do_pending_stack_adjust (); - emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from), - 0, lab1); - - /* If not, do the signed "fix" and branch around fixup code. */ - expand_fix (to, from, 0); - emit_jump_insn (gen_jump (lab2)); - emit_barrier (); - - /* Otherwise, subtract 2**(N-1), convert to signed number, - then add 2**(N-1). Do the addition using XOR since this - will often generate better code. */ - emit_label (lab1); - target = expand_binop (GET_MODE (from), sub_optab, from, limit, - NULL_RTX, 0, OPTAB_LIB_WIDEN); - expand_fix (to, target, 0); - target = expand_binop (GET_MODE (to), xor_optab, to, - gen_int_mode - ((HOST_WIDE_INT) 1 << (bitsize - 1), - GET_MODE (to)), - to, 1, OPTAB_LIB_WIDEN); - - if (target != to) - emit_move_insn (to, target); - - emit_label (lab2); - - if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing) - { - /* Make a place for a REG_NOTE and add it. */ - insn = emit_move_insn (to, to); - set_unique_reg_note (insn, - REG_EQUAL, - gen_rtx_fmt_e (UNSIGNED_FIX, - GET_MODE (to), - copy_rtx (from))); - } - - return; - } + scalar_int_mode to_mode; + if (unsignedp + && is_a <scalar_int_mode> (GET_MODE (to), &to_mode) + && HWI_COMPUTABLE_MODE_P (to_mode)) + FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from))) + { + scalar_mode fmode = fmode_iter.require (); + if (CODE_FOR_nothing != can_fix_p (to_mode, fmode, + 0, &must_trunc) + && (!DECIMAL_FLOAT_MODE_P (fmode) + || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode)))) + { + int bitsize; + REAL_VALUE_TYPE offset; + rtx limit; + rtx_code_label *lab1, *lab2; + rtx_insn *insn; + + bitsize = GET_MODE_PRECISION (to_mode); + real_2expN (&offset, bitsize - 1, fmode); + limit = const_double_from_real_value (offset, fmode); + lab1 = gen_label_rtx (); + lab2 = gen_label_rtx (); + + if (fmode != GET_MODE (from)) + from = convert_to_mode (fmode, from, 0); + + /* See if we need to do the subtraction. */ + do_pending_stack_adjust (); + emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, + GET_MODE (from), 0, lab1); + + /* If not, do the signed "fix" and branch around fixup code. */ + expand_fix (to, from, 0); + emit_jump_insn (targetm.gen_jump (lab2)); + emit_barrier (); + + /* Otherwise, subtract 2**(N-1), convert to signed number, + then add 2**(N-1). Do the addition using XOR since this + will often generate better code. */ + emit_label (lab1); + target = expand_binop (GET_MODE (from), sub_optab, from, limit, + NULL_RTX, 0, OPTAB_LIB_WIDEN); + expand_fix (to, target, 0); + target = expand_binop (to_mode, xor_optab, to, + gen_int_mode + (HOST_WIDE_INT_1 << (bitsize - 1), + to_mode), + to, 1, OPTAB_LIB_WIDEN); + + if (target != to) + emit_move_insn (to, target); + + emit_label (lab2); + + if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing) + { + /* Make a place for a REG_NOTE and add it. */ + insn = emit_move_insn (to, to); + set_dst_reg_note (insn, REG_EQUAL, + gen_rtx_fmt_e (UNSIGNED_FIX, to_mode, + copy_rtx (from)), + to); + } + + return; + } + } /* We can't do it with an insn, so use a library call. But first ensure that the mode of TO is at least as wide as SImode, since those are the only library calls we know about. */ - if (GET_MODE_SIZE (GET_MODE (to)) < GET_MODE_SIZE (SImode)) + if (GET_MODE_PRECISION (GET_MODE (to)) < GET_MODE_PRECISION (SImode)) { target = gen_reg_rtx (SImode); @@ -5265,7 +4993,7 @@ } else { - rtx insns; + rtx_insn *insns; rtx value; rtx libfunc; @@ -5276,8 +5004,7 @@ start_sequence (); value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, - GET_MODE (to), 1, from, - GET_MODE (from)); + GET_MODE (to), from, GET_MODE (from)); insns = get_insns (); end_sequence (); @@ -5295,6 +5022,33 @@ } } + +/* Promote integer arguments for a libcall if necessary. + emit_library_call_value cannot do the promotion because it does not + know if it should do a signed or unsigned promotion. This is because + there are no tree types defined for libcalls. */ + +static rtx +prepare_libcall_arg (rtx arg, int uintp) +{ + scalar_int_mode mode; + machine_mode arg_mode; + if (is_a <scalar_int_mode> (GET_MODE (arg), &mode)) + { + /* If we need to promote the integer function argument we need to do + it here instead of inside emit_library_call_value because in + emit_library_call_value we don't know if we should do a signed or + unsigned promotion. */ + + int unsigned_p = 0; + arg_mode = promote_function_mode (NULL_TREE, mode, + &unsigned_p, NULL_TREE, 0); + if (arg_mode != mode) + return convert_to_mode (arg_mode, arg, uintp); + } + return arg; +} + /* Generate code to convert FROM or TO a fixed-point. If UINTP is true, either TO or FROM is an unsigned integer. If SATP is true, we need to saturate the result. */ @@ -5302,12 +5056,13 @@ void expand_fixed_convert (rtx to, rtx from, int uintp, int satp) { - enum machine_mode to_mode = GET_MODE (to); - enum machine_mode from_mode = GET_MODE (from); + machine_mode to_mode = GET_MODE (to); + machine_mode from_mode = GET_MODE (from); convert_optab tab; enum rtx_code this_code; enum insn_code code; - rtx insns, value; + rtx_insn *insns; + rtx value; rtx libfunc; if (to_mode == from_mode) @@ -5336,14 +5091,17 @@ libfunc = convert_optab_libfunc (tab, to_mode, from_mode); gcc_assert (libfunc); + from = prepare_libcall_arg (from, uintp); + from_mode = GET_MODE (from); + start_sequence (); value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode, - 1, from, from_mode); + from, from_mode); insns = get_insns (); end_sequence (); emit_libcall_block (insns, to, value, - gen_rtx_fmt_e (tab->code, to_mode, from)); + gen_rtx_fmt_e (optab_to_code (tab), to_mode, from)); } /* Generate code to convert FROM to fixed point and store in TO. FROM @@ -5355,22 +5113,20 @@ { enum insn_code icode; rtx target = to; - enum machine_mode fmode, imode; + machine_mode fmode, imode; /* We first try to find a pair of modes, one real and one integer, at least as wide as FROM and TO, respectively, in which we can open-code this conversion. If the integer mode is wider than the mode of TO, we can do the conversion either signed or unsigned. */ - for (fmode = GET_MODE (from); fmode != VOIDmode; - fmode = GET_MODE_WIDER_MODE (fmode)) - for (imode = GET_MODE (to); imode != VOIDmode; - imode = GET_MODE_WIDER_MODE (imode)) + FOR_EACH_MODE_FROM (fmode, GET_MODE (from)) + FOR_EACH_MODE_FROM (imode, GET_MODE (to)) { icode = convert_optab_handler (tab, imode, fmode); if (icode != CODE_FOR_nothing) { - rtx last = get_last_insn (); + rtx_insn *last = get_last_insn (); if (fmode != GET_MODE (from)) from = convert_to_mode (fmode, from, 0); @@ -5394,1181 +5150,11 @@ /* Report whether we have an instruction to perform the operation specified by CODE on operands of mode MODE. */ int -have_insn_for (enum rtx_code code, enum machine_mode mode) -{ - return (code_to_optab[(int) code] != 0 - && (optab_handler (code_to_optab[(int) code], mode) - != CODE_FOR_nothing)); -} - -/* Set all insn_code fields to CODE_FOR_nothing. */ - -static void -init_insn_codes (void) -{ - memset (optab_table, 0, sizeof (optab_table)); - memset (convert_optab_table, 0, sizeof (convert_optab_table)); - memset (direct_optab_table, 0, sizeof (direct_optab_table)); -} - -/* Initialize OP's code to CODE, and write it into the code_to_optab table. */ -static inline void -init_optab (optab op, enum rtx_code code) -{ - op->code = code; - code_to_optab[(int) code] = op; -} - -/* Same, but fill in its code as CODE, and do _not_ write it into - the code_to_optab table. */ -static inline void -init_optabv (optab op, enum rtx_code code) -{ - op->code = code; -} - -/* Conversion optabs never go in the code_to_optab table. */ -static void -init_convert_optab (convert_optab op, enum rtx_code code) -{ - op->code = code; -} - -/* Initialize the libfunc fields of an entire group of entries in some - optab. Each entry is set equal to a string consisting of a leading - pair of underscores followed by a generic operation name followed by - a mode name (downshifted to lowercase) followed by a single character - representing the number of operands for the given operation (which is - usually one of the characters '2', '3', or '4'). - - OPTABLE is the table in which libfunc fields are to be initialized. - OPNAME is the generic (string) name of the operation. - SUFFIX is the character which specifies the number of operands for - the given generic operation. - MODE is the mode to generate for. -*/ - -static void -gen_libfunc (optab optable, const char *opname, int suffix, enum machine_mode mode) -{ - unsigned opname_len = strlen (opname); - const char *mname = GET_MODE_NAME (mode); - unsigned mname_len = strlen (mname); - char *libfunc_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1); - char *p; - const char *q; - - p = libfunc_name; - *p++ = '_'; - *p++ = '_'; - for (q = opname; *q; ) - *p++ = *q++; - for (q = mname; *q; q++) - *p++ = TOLOWER (*q); - *p++ = suffix; - *p = '\0'; - - set_optab_libfunc (optable, mode, - ggc_alloc_string (libfunc_name, p - libfunc_name)); -} - -/* Like gen_libfunc, but verify that integer operation is involved. */ - -static void -gen_int_libfunc (optab optable, const char *opname, char suffix, - enum machine_mode mode) -{ - int maxsize = 2 * BITS_PER_WORD; - - if (GET_MODE_CLASS (mode) != MODE_INT) - return; - if (maxsize < LONG_LONG_TYPE_SIZE) - maxsize = LONG_LONG_TYPE_SIZE; - if (GET_MODE_CLASS (mode) != MODE_INT - || mode < word_mode || GET_MODE_BITSIZE (mode) > maxsize) - return; - gen_libfunc (optable, opname, suffix, mode); -} - -/* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */ - -static void -gen_fp_libfunc (optab optable, const char *opname, char suffix, - enum machine_mode mode) -{ - char *dec_opname; - - if (GET_MODE_CLASS (mode) == MODE_FLOAT) - gen_libfunc (optable, opname, suffix, mode); - if (DECIMAL_FLOAT_MODE_P (mode)) - { - dec_opname = XALLOCAVEC (char, sizeof (DECIMAL_PREFIX) + strlen (opname)); - /* For BID support, change the name to have either a bid_ or dpd_ prefix - depending on the low level floating format used. */ - memcpy (dec_opname, DECIMAL_PREFIX, sizeof (DECIMAL_PREFIX) - 1); - strcpy (dec_opname + sizeof (DECIMAL_PREFIX) - 1, opname); - gen_libfunc (optable, dec_opname, suffix, mode); - } -} - -/* Like gen_libfunc, but verify that fixed-point operation is involved. */ - -static void -gen_fixed_libfunc (optab optable, const char *opname, char suffix, - enum machine_mode mode) -{ - if (!ALL_FIXED_POINT_MODE_P (mode)) - return; - gen_libfunc (optable, opname, suffix, mode); -} - -/* Like gen_libfunc, but verify that signed fixed-point operation is - involved. */ - -static void -gen_signed_fixed_libfunc (optab optable, const char *opname, char suffix, - enum machine_mode mode) -{ - if (!SIGNED_FIXED_POINT_MODE_P (mode)) - return; - gen_libfunc (optable, opname, suffix, mode); -} - -/* Like gen_libfunc, but verify that unsigned fixed-point operation is - involved. */ - -static void -gen_unsigned_fixed_libfunc (optab optable, const char *opname, char suffix, - enum machine_mode mode) -{ - if (!UNSIGNED_FIXED_POINT_MODE_P (mode)) - return; - gen_libfunc (optable, opname, suffix, mode); -} - -/* Like gen_libfunc, but verify that FP or INT operation is involved. */ - -static void -gen_int_fp_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT) - gen_fp_libfunc (optable, name, suffix, mode); - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); -} - -/* Like gen_libfunc, but verify that FP or INT operation is involved - and add 'v' suffix for integer operation. */ - -static void -gen_intv_fp_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT) - gen_fp_libfunc (optable, name, suffix, mode); - if (GET_MODE_CLASS (mode) == MODE_INT) - { - int len = strlen (name); - char *v_name = XALLOCAVEC (char, len + 2); - strcpy (v_name, name); - v_name[len] = 'v'; - v_name[len + 1] = 0; - gen_int_libfunc (optable, v_name, suffix, mode); - } -} - -/* Like gen_libfunc, but verify that FP or INT or FIXED operation is - involved. */ - -static void -gen_int_fp_fixed_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT) - gen_fp_libfunc (optable, name, suffix, mode); - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); - if (ALL_FIXED_POINT_MODE_P (mode)) - gen_fixed_libfunc (optable, name, suffix, mode); -} - -/* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is - involved. */ - -static void -gen_int_fp_signed_fixed_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT) - gen_fp_libfunc (optable, name, suffix, mode); - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); - if (SIGNED_FIXED_POINT_MODE_P (mode)) - gen_signed_fixed_libfunc (optable, name, suffix, mode); -} - -/* Like gen_libfunc, but verify that INT or FIXED operation is - involved. */ - -static void -gen_int_fixed_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); - if (ALL_FIXED_POINT_MODE_P (mode)) - gen_fixed_libfunc (optable, name, suffix, mode); -} - -/* Like gen_libfunc, but verify that INT or signed FIXED operation is - involved. */ - -static void -gen_int_signed_fixed_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); - if (SIGNED_FIXED_POINT_MODE_P (mode)) - gen_signed_fixed_libfunc (optable, name, suffix, mode); -} - -/* Like gen_libfunc, but verify that INT or unsigned FIXED operation is - involved. */ - -static void -gen_int_unsigned_fixed_libfunc (optab optable, const char *name, char suffix, - enum machine_mode mode) -{ - if (INTEGRAL_MODE_P (mode)) - gen_int_libfunc (optable, name, suffix, mode); - if (UNSIGNED_FIXED_POINT_MODE_P (mode)) - gen_unsigned_fixed_libfunc (optable, name, suffix, mode); -} - -/* Initialize the libfunc fields of an entire group of entries of an - inter-mode-class conversion optab. The string formation rules are - similar to the ones for init_libfuncs, above, but instead of having - a mode name and an operand count these functions have two mode names - and no operand count. */ - -static void -gen_interclass_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - size_t opname_len = strlen (opname); - size_t mname_len = 0; - - const char *fname, *tname; - const char *q; - char *libfunc_name, *suffix; - char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix; - char *p; - - /* If this is a decimal conversion, add the current BID vs. DPD prefix that - depends on which underlying decimal floating point format is used. */ - const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1; - - mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode)); - - nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1); - nondec_name[0] = '_'; - nondec_name[1] = '_'; - memcpy (&nondec_name[2], opname, opname_len); - nondec_suffix = nondec_name + opname_len + 2; - - dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1); - dec_name[0] = '_'; - dec_name[1] = '_'; - memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len); - memcpy (&dec_name[2+dec_len], opname, opname_len); - dec_suffix = dec_name + dec_len + opname_len + 2; - - fname = GET_MODE_NAME (fmode); - tname = GET_MODE_NAME (tmode); - - if (DECIMAL_FLOAT_MODE_P(fmode) || DECIMAL_FLOAT_MODE_P(tmode)) - { - libfunc_name = dec_name; - suffix = dec_suffix; - } - else - { - libfunc_name = nondec_name; - suffix = nondec_suffix; - } - - p = suffix; - for (q = fname; *q; p++, q++) - *p = TOLOWER (*q); - for (q = tname; *q; p++, q++) - *p = TOLOWER (*q); - - *p = '\0'; - - set_conv_libfunc (tab, tmode, fmode, - ggc_alloc_string (libfunc_name, p - libfunc_name)); -} - -/* Same as gen_interclass_conv_libfunc but verify that we are producing - int->fp conversion. */ - -static void -gen_int_to_fp_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (GET_MODE_CLASS (fmode) != MODE_INT) - return; - if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode)) - return; - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* ufloat_optab is special by using floatun for FP and floatuns decimal fp - naming scheme. */ - -static void -gen_ufloat_conv_libfunc (convert_optab tab, - const char *opname ATTRIBUTE_UNUSED, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (DECIMAL_FLOAT_MODE_P (tmode)) - gen_int_to_fp_conv_libfunc (tab, "floatuns", tmode, fmode); - else - gen_int_to_fp_conv_libfunc (tab, "floatun", tmode, fmode); -} - -/* Same as gen_interclass_conv_libfunc but verify that we are producing - fp->int conversion. */ - -static void -gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (GET_MODE_CLASS (fmode) != MODE_INT) - return; - if (GET_MODE_CLASS (tmode) != MODE_FLOAT) - return; - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Same as gen_interclass_conv_libfunc but verify that we are producing - fp->int conversion with no decimal floating point involved. */ - -static void -gen_fp_to_int_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode)) - return; - if (GET_MODE_CLASS (tmode) != MODE_INT) - return; - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Initialize the libfunc fields of an of an intra-mode-class conversion optab. - The string formation rules are - similar to the ones for init_libfunc, above. */ - -static void -gen_intraclass_conv_libfunc (convert_optab tab, const char *opname, - enum machine_mode tmode, enum machine_mode fmode) -{ - size_t opname_len = strlen (opname); - size_t mname_len = 0; - - const char *fname, *tname; - const char *q; - char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix; - char *libfunc_name, *suffix; - char *p; - - /* If this is a decimal conversion, add the current BID vs. DPD prefix that - depends on which underlying decimal floating point format is used. */ - const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1; - - mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode)); - - nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1); - nondec_name[0] = '_'; - nondec_name[1] = '_'; - memcpy (&nondec_name[2], opname, opname_len); - nondec_suffix = nondec_name + opname_len + 2; - - dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1); - dec_name[0] = '_'; - dec_name[1] = '_'; - memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len); - memcpy (&dec_name[2 + dec_len], opname, opname_len); - dec_suffix = dec_name + dec_len + opname_len + 2; - - fname = GET_MODE_NAME (fmode); - tname = GET_MODE_NAME (tmode); - - if (DECIMAL_FLOAT_MODE_P(fmode) || DECIMAL_FLOAT_MODE_P(tmode)) - { - libfunc_name = dec_name; - suffix = dec_suffix; - } - else - { - libfunc_name = nondec_name; - suffix = nondec_suffix; - } - - p = suffix; - for (q = fname; *q; p++, q++) - *p = TOLOWER (*q); - for (q = tname; *q; p++, q++) - *p = TOLOWER (*q); - - *p++ = '2'; - *p = '\0'; - - set_conv_libfunc (tab, tmode, fmode, - ggc_alloc_string (libfunc_name, p - libfunc_name)); -} - -/* Pick proper libcall for trunc_optab. We need to chose if we do - truncation or extension and interclass or intraclass. */ - -static void -gen_trunc_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode)) - return; - if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode)) - return; - if (tmode == fmode) - return; - - if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode)) - || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode))) - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); - - if (GET_MODE_PRECISION (fmode) <= GET_MODE_PRECISION (tmode)) - return; - - if ((GET_MODE_CLASS (tmode) == MODE_FLOAT - && GET_MODE_CLASS (fmode) == MODE_FLOAT) - || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode))) - gen_intraclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Pick proper libcall for extend_optab. We need to chose if we do - truncation or extension and interclass or intraclass. */ - -static void -gen_extend_conv_libfunc (convert_optab tab, - const char *opname ATTRIBUTE_UNUSED, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode)) - return; - if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode)) - return; - if (tmode == fmode) - return; - - if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode)) - || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode))) - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); - - if (GET_MODE_PRECISION (fmode) > GET_MODE_PRECISION (tmode)) - return; - - if ((GET_MODE_CLASS (tmode) == MODE_FLOAT - && GET_MODE_CLASS (fmode) == MODE_FLOAT) - || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode))) - gen_intraclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Pick proper libcall for fract_optab. We need to chose if we do - interclass or intraclass. */ - -static void -gen_fract_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (tmode == fmode) - return; - if (!(ALL_FIXED_POINT_MODE_P (tmode) || ALL_FIXED_POINT_MODE_P (fmode))) - return; - - if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode)) - gen_intraclass_conv_libfunc (tab, opname, tmode, fmode); - else - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Pick proper libcall for fractuns_optab. */ - -static void -gen_fractuns_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (tmode == fmode) - return; - /* One mode must be a fixed-point mode, and the other must be an integer - mode. */ - if (!((ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT) - || (ALL_FIXED_POINT_MODE_P (fmode) - && GET_MODE_CLASS (tmode) == MODE_INT))) - return; - - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Pick proper libcall for satfract_optab. We need to chose if we do - interclass or intraclass. */ - -static void -gen_satfract_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) -{ - if (tmode == fmode) - return; - /* TMODE must be a fixed-point mode. */ - if (!ALL_FIXED_POINT_MODE_P (tmode)) - return; - - if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode)) - gen_intraclass_conv_libfunc (tab, opname, tmode, fmode); - else - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* Pick proper libcall for satfractuns_optab. */ - -static void -gen_satfractuns_conv_libfunc (convert_optab tab, - const char *opname, - enum machine_mode tmode, - enum machine_mode fmode) +have_insn_for (enum rtx_code code, machine_mode mode) { - if (tmode == fmode) - return; - /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */ - if (!(ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT)) - return; - - gen_interclass_conv_libfunc (tab, opname, tmode, fmode); -} - -/* A table of previously-created libfuncs, hashed by name. */ -static GTY ((param_is (union tree_node))) htab_t libfunc_decls; - -/* Hashtable callbacks for libfunc_decls. */ - -static hashval_t -libfunc_decl_hash (const void *entry) -{ - return IDENTIFIER_HASH_VALUE (DECL_NAME ((const_tree) entry)); -} - -static int -libfunc_decl_eq (const void *entry1, const void *entry2) -{ - return DECL_NAME ((const_tree) entry1) == (const_tree) entry2; -} - -/* Build a decl for a libfunc named NAME. */ - -tree -build_libfunc_function (const char *name) -{ - tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, - get_identifier (name), - build_function_type (integer_type_node, NULL_TREE)); - /* ??? We don't have any type information except for this is - a function. Pretend this is "int foo()". */ - DECL_ARTIFICIAL (decl) = 1; - DECL_EXTERNAL (decl) = 1; - TREE_PUBLIC (decl) = 1; - gcc_assert (DECL_ASSEMBLER_NAME (decl)); - - /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with - are the flags assigned by targetm.encode_section_info. */ - SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL); - - return decl; -} - -rtx -init_one_libfunc (const char *name) -{ - tree id, decl; - void **slot; - hashval_t hash; - - if (libfunc_decls == NULL) - libfunc_decls = htab_create_ggc (37, libfunc_decl_hash, - libfunc_decl_eq, NULL); - - /* See if we have already created a libfunc decl for this function. */ - id = get_identifier (name); - hash = IDENTIFIER_HASH_VALUE (id); - slot = htab_find_slot_with_hash (libfunc_decls, id, hash, INSERT); - decl = (tree) *slot; - if (decl == NULL) - { - /* Create a new decl, so that it can be passed to - targetm.encode_section_info. */ - decl = build_libfunc_function (name); - *slot = decl; - } - return XEXP (DECL_RTL (decl), 0); -} - -/* Adjust the assembler name of libfunc NAME to ASMSPEC. */ - -rtx -set_user_assembler_libfunc (const char *name, const char *asmspec) -{ - tree id, decl; - void **slot; - hashval_t hash; - - id = get_identifier (name); - hash = IDENTIFIER_HASH_VALUE (id); - slot = htab_find_slot_with_hash (libfunc_decls, id, hash, NO_INSERT); - gcc_assert (slot); - decl = (tree) *slot; - set_user_assembler_name (decl, asmspec); - return XEXP (DECL_RTL (decl), 0); -} - -/* Call this to reset the function entry for one optab (OPTABLE) in mode - MODE to NAME, which should be either 0 or a string constant. */ -void -set_optab_libfunc (optab optable, enum machine_mode mode, const char *name) -{ - rtx val; - struct libfunc_entry e; - struct libfunc_entry **slot; - e.optab = (size_t) (optable - &optab_table[0]); - e.mode1 = mode; - e.mode2 = VOIDmode; - - if (name) - val = init_one_libfunc (name); - else - val = 0; - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT); - if (*slot == NULL) - *slot = ggc_alloc_libfunc_entry (); - (*slot)->optab = (size_t) (optable - &optab_table[0]); - (*slot)->mode1 = mode; - (*slot)->mode2 = VOIDmode; - (*slot)->libfunc = val; -} - -/* Call this to reset the function entry for one conversion optab - (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be - either 0 or a string constant. */ -void -set_conv_libfunc (convert_optab optable, enum machine_mode tmode, - enum machine_mode fmode, const char *name) -{ - rtx val; - struct libfunc_entry e; - struct libfunc_entry **slot; - e.optab = (size_t) (optable - &convert_optab_table[0]); - e.mode1 = tmode; - e.mode2 = fmode; - - if (name) - val = init_one_libfunc (name); - else - val = 0; - slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT); - if (*slot == NULL) - *slot = ggc_alloc_libfunc_entry (); - (*slot)->optab = (size_t) (optable - &convert_optab_table[0]); - (*slot)->mode1 = tmode; - (*slot)->mode2 = fmode; - (*slot)->libfunc = val; -} - -/* Call this to initialize the contents of the optabs - appropriately for the current target machine. */ - -void -init_optabs (void) -{ - if (libfunc_hash) - { - htab_empty (libfunc_hash); - /* We statically initialize the insn_codes with the equivalent of - CODE_FOR_nothing. Repeat the process if reinitialising. */ - init_insn_codes (); - } - else - libfunc_hash = htab_create_ggc (10, hash_libfunc, eq_libfunc, NULL); - - init_optab (add_optab, PLUS); - init_optabv (addv_optab, PLUS); - init_optab (sub_optab, MINUS); - init_optabv (subv_optab, MINUS); - init_optab (ssadd_optab, SS_PLUS); - init_optab (usadd_optab, US_PLUS); - init_optab (sssub_optab, SS_MINUS); - init_optab (ussub_optab, US_MINUS); - init_optab (smul_optab, MULT); - init_optab (ssmul_optab, SS_MULT); - init_optab (usmul_optab, US_MULT); - init_optabv (smulv_optab, MULT); - init_optab (smul_highpart_optab, UNKNOWN); - init_optab (umul_highpart_optab, UNKNOWN); - init_optab (smul_widen_optab, UNKNOWN); - init_optab (umul_widen_optab, UNKNOWN); - init_optab (usmul_widen_optab, UNKNOWN); - init_optab (smadd_widen_optab, UNKNOWN); - init_optab (umadd_widen_optab, UNKNOWN); - init_optab (ssmadd_widen_optab, UNKNOWN); - init_optab (usmadd_widen_optab, UNKNOWN); - init_optab (smsub_widen_optab, UNKNOWN); - init_optab (umsub_widen_optab, UNKNOWN); - init_optab (ssmsub_widen_optab, UNKNOWN); - init_optab (usmsub_widen_optab, UNKNOWN); - init_optab (sdiv_optab, DIV); - init_optab (ssdiv_optab, SS_DIV); - init_optab (usdiv_optab, US_DIV); - init_optabv (sdivv_optab, DIV); - init_optab (sdivmod_optab, UNKNOWN); - init_optab (udiv_optab, UDIV); - init_optab (udivmod_optab, UNKNOWN); - init_optab (smod_optab, MOD); - init_optab (umod_optab, UMOD); - init_optab (fmod_optab, UNKNOWN); - init_optab (remainder_optab, UNKNOWN); - init_optab (ftrunc_optab, UNKNOWN); - init_optab (and_optab, AND); - init_optab (ior_optab, IOR); - init_optab (xor_optab, XOR); - init_optab (ashl_optab, ASHIFT); - init_optab (ssashl_optab, SS_ASHIFT); - init_optab (usashl_optab, US_ASHIFT); - init_optab (ashr_optab, ASHIFTRT); - init_optab (lshr_optab, LSHIFTRT); - init_optab (rotl_optab, ROTATE); - init_optab (rotr_optab, ROTATERT); - init_optab (smin_optab, SMIN); - init_optab (smax_optab, SMAX); - init_optab (umin_optab, UMIN); - init_optab (umax_optab, UMAX); - init_optab (pow_optab, UNKNOWN); - init_optab (atan2_optab, UNKNOWN); - init_optab (fma_optab, FMA); - init_optab (fms_optab, UNKNOWN); - init_optab (fnma_optab, UNKNOWN); - init_optab (fnms_optab, UNKNOWN); - - /* These three have codes assigned exclusively for the sake of - have_insn_for. */ - init_optab (mov_optab, SET); - init_optab (movstrict_optab, STRICT_LOW_PART); - init_optab (cbranch_optab, COMPARE); - - init_optab (cmov_optab, UNKNOWN); - init_optab (cstore_optab, UNKNOWN); - init_optab (ctrap_optab, UNKNOWN); - - init_optab (storent_optab, UNKNOWN); - - init_optab (cmp_optab, UNKNOWN); - init_optab (ucmp_optab, UNKNOWN); - - init_optab (eq_optab, EQ); - init_optab (ne_optab, NE); - init_optab (gt_optab, GT); - init_optab (ge_optab, GE); - init_optab (lt_optab, LT); - init_optab (le_optab, LE); - init_optab (unord_optab, UNORDERED); - - init_optab (neg_optab, NEG); - init_optab (ssneg_optab, SS_NEG); - init_optab (usneg_optab, US_NEG); - init_optabv (negv_optab, NEG); - init_optab (abs_optab, ABS); - init_optabv (absv_optab, ABS); - init_optab (addcc_optab, UNKNOWN); - init_optab (one_cmpl_optab, NOT); - init_optab (bswap_optab, BSWAP); - init_optab (ffs_optab, FFS); - init_optab (clz_optab, CLZ); - init_optab (ctz_optab, CTZ); - init_optab (popcount_optab, POPCOUNT); - init_optab (parity_optab, PARITY); - init_optab (sqrt_optab, SQRT); - init_optab (floor_optab, UNKNOWN); - init_optab (ceil_optab, UNKNOWN); - init_optab (round_optab, UNKNOWN); - init_optab (btrunc_optab, UNKNOWN); - init_optab (nearbyint_optab, UNKNOWN); - init_optab (rint_optab, UNKNOWN); - init_optab (sincos_optab, UNKNOWN); - init_optab (sin_optab, UNKNOWN); - init_optab (asin_optab, UNKNOWN); - init_optab (cos_optab, UNKNOWN); - init_optab (acos_optab, UNKNOWN); - init_optab (exp_optab, UNKNOWN); - init_optab (exp10_optab, UNKNOWN); - init_optab (exp2_optab, UNKNOWN); - init_optab (expm1_optab, UNKNOWN); - init_optab (ldexp_optab, UNKNOWN); - init_optab (scalb_optab, UNKNOWN); - init_optab (significand_optab, UNKNOWN); - init_optab (logb_optab, UNKNOWN); - init_optab (ilogb_optab, UNKNOWN); - init_optab (log_optab, UNKNOWN); - init_optab (log10_optab, UNKNOWN); - init_optab (log2_optab, UNKNOWN); - init_optab (log1p_optab, UNKNOWN); - init_optab (tan_optab, UNKNOWN); - init_optab (atan_optab, UNKNOWN); - init_optab (copysign_optab, UNKNOWN); - init_optab (signbit_optab, UNKNOWN); - - init_optab (isinf_optab, UNKNOWN); - - init_optab (strlen_optab, UNKNOWN); - init_optab (push_optab, UNKNOWN); - - init_optab (reduc_smax_optab, UNKNOWN); - init_optab (reduc_umax_optab, UNKNOWN); - init_optab (reduc_smin_optab, UNKNOWN); - init_optab (reduc_umin_optab, UNKNOWN); - init_optab (reduc_splus_optab, UNKNOWN); - init_optab (reduc_uplus_optab, UNKNOWN); - - init_optab (ssum_widen_optab, UNKNOWN); - init_optab (usum_widen_optab, UNKNOWN); - init_optab (sdot_prod_optab, UNKNOWN); - init_optab (udot_prod_optab, UNKNOWN); - - init_optab (vec_extract_optab, UNKNOWN); - init_optab (vec_extract_even_optab, UNKNOWN); - init_optab (vec_extract_odd_optab, UNKNOWN); - init_optab (vec_interleave_high_optab, UNKNOWN); - init_optab (vec_interleave_low_optab, UNKNOWN); - init_optab (vec_set_optab, UNKNOWN); - init_optab (vec_init_optab, UNKNOWN); - init_optab (vec_shl_optab, UNKNOWN); - init_optab (vec_shr_optab, UNKNOWN); - init_optab (vec_realign_load_optab, UNKNOWN); - init_optab (movmisalign_optab, UNKNOWN); - init_optab (vec_widen_umult_hi_optab, UNKNOWN); - init_optab (vec_widen_umult_lo_optab, UNKNOWN); - init_optab (vec_widen_smult_hi_optab, UNKNOWN); - init_optab (vec_widen_smult_lo_optab, UNKNOWN); - init_optab (vec_unpacks_hi_optab, UNKNOWN); - init_optab (vec_unpacks_lo_optab, UNKNOWN); - init_optab (vec_unpacku_hi_optab, UNKNOWN); - init_optab (vec_unpacku_lo_optab, UNKNOWN); - init_optab (vec_unpacks_float_hi_optab, UNKNOWN); - init_optab (vec_unpacks_float_lo_optab, UNKNOWN); - init_optab (vec_unpacku_float_hi_optab, UNKNOWN); - init_optab (vec_unpacku_float_lo_optab, UNKNOWN); - init_optab (vec_pack_trunc_optab, UNKNOWN); - init_optab (vec_pack_usat_optab, UNKNOWN); - init_optab (vec_pack_ssat_optab, UNKNOWN); - init_optab (vec_pack_ufix_trunc_optab, UNKNOWN); - init_optab (vec_pack_sfix_trunc_optab, UNKNOWN); - - init_optab (powi_optab, UNKNOWN); - - /* Conversions. */ - init_convert_optab (sext_optab, SIGN_EXTEND); - init_convert_optab (zext_optab, ZERO_EXTEND); - init_convert_optab (trunc_optab, TRUNCATE); - init_convert_optab (sfix_optab, FIX); - init_convert_optab (ufix_optab, UNSIGNED_FIX); - init_convert_optab (sfixtrunc_optab, UNKNOWN); - init_convert_optab (ufixtrunc_optab, UNKNOWN); - init_convert_optab (sfloat_optab, FLOAT); - init_convert_optab (ufloat_optab, UNSIGNED_FLOAT); - init_convert_optab (lrint_optab, UNKNOWN); - init_convert_optab (lround_optab, UNKNOWN); - init_convert_optab (lfloor_optab, UNKNOWN); - init_convert_optab (lceil_optab, UNKNOWN); - - init_convert_optab (fract_optab, FRACT_CONVERT); - init_convert_optab (fractuns_optab, UNSIGNED_FRACT_CONVERT); - init_convert_optab (satfract_optab, SAT_FRACT); - init_convert_optab (satfractuns_optab, UNSIGNED_SAT_FRACT); - - /* Fill in the optabs with the insns we support. */ - init_all_optabs (); - - /* Initialize the optabs with the names of the library functions. */ - add_optab->libcall_basename = "add"; - add_optab->libcall_suffix = '3'; - add_optab->libcall_gen = gen_int_fp_fixed_libfunc; - addv_optab->libcall_basename = "add"; - addv_optab->libcall_suffix = '3'; - addv_optab->libcall_gen = gen_intv_fp_libfunc; - ssadd_optab->libcall_basename = "ssadd"; - ssadd_optab->libcall_suffix = '3'; - ssadd_optab->libcall_gen = gen_signed_fixed_libfunc; - usadd_optab->libcall_basename = "usadd"; - usadd_optab->libcall_suffix = '3'; - usadd_optab->libcall_gen = gen_unsigned_fixed_libfunc; - sub_optab->libcall_basename = "sub"; - sub_optab->libcall_suffix = '3'; - sub_optab->libcall_gen = gen_int_fp_fixed_libfunc; - subv_optab->libcall_basename = "sub"; - subv_optab->libcall_suffix = '3'; - subv_optab->libcall_gen = gen_intv_fp_libfunc; - sssub_optab->libcall_basename = "sssub"; - sssub_optab->libcall_suffix = '3'; - sssub_optab->libcall_gen = gen_signed_fixed_libfunc; - ussub_optab->libcall_basename = "ussub"; - ussub_optab->libcall_suffix = '3'; - ussub_optab->libcall_gen = gen_unsigned_fixed_libfunc; - smul_optab->libcall_basename = "mul"; - smul_optab->libcall_suffix = '3'; - smul_optab->libcall_gen = gen_int_fp_fixed_libfunc; - smulv_optab->libcall_basename = "mul"; - smulv_optab->libcall_suffix = '3'; - smulv_optab->libcall_gen = gen_intv_fp_libfunc; - ssmul_optab->libcall_basename = "ssmul"; - ssmul_optab->libcall_suffix = '3'; - ssmul_optab->libcall_gen = gen_signed_fixed_libfunc; - usmul_optab->libcall_basename = "usmul"; - usmul_optab->libcall_suffix = '3'; - usmul_optab->libcall_gen = gen_unsigned_fixed_libfunc; - sdiv_optab->libcall_basename = "div"; - sdiv_optab->libcall_suffix = '3'; - sdiv_optab->libcall_gen = gen_int_fp_signed_fixed_libfunc; - sdivv_optab->libcall_basename = "divv"; - sdivv_optab->libcall_suffix = '3'; - sdivv_optab->libcall_gen = gen_int_libfunc; - ssdiv_optab->libcall_basename = "ssdiv"; - ssdiv_optab->libcall_suffix = '3'; - ssdiv_optab->libcall_gen = gen_signed_fixed_libfunc; - udiv_optab->libcall_basename = "udiv"; - udiv_optab->libcall_suffix = '3'; - udiv_optab->libcall_gen = gen_int_unsigned_fixed_libfunc; - usdiv_optab->libcall_basename = "usdiv"; - usdiv_optab->libcall_suffix = '3'; - usdiv_optab->libcall_gen = gen_unsigned_fixed_libfunc; - sdivmod_optab->libcall_basename = "divmod"; - sdivmod_optab->libcall_suffix = '4'; - sdivmod_optab->libcall_gen = gen_int_libfunc; - udivmod_optab->libcall_basename = "udivmod"; - udivmod_optab->libcall_suffix = '4'; - udivmod_optab->libcall_gen = gen_int_libfunc; - smod_optab->libcall_basename = "mod"; - smod_optab->libcall_suffix = '3'; - smod_optab->libcall_gen = gen_int_libfunc; - umod_optab->libcall_basename = "umod"; - umod_optab->libcall_suffix = '3'; - umod_optab->libcall_gen = gen_int_libfunc; - ftrunc_optab->libcall_basename = "ftrunc"; - ftrunc_optab->libcall_suffix = '2'; - ftrunc_optab->libcall_gen = gen_fp_libfunc; - and_optab->libcall_basename = "and"; - and_optab->libcall_suffix = '3'; - and_optab->libcall_gen = gen_int_libfunc; - ior_optab->libcall_basename = "ior"; - ior_optab->libcall_suffix = '3'; - ior_optab->libcall_gen = gen_int_libfunc; - xor_optab->libcall_basename = "xor"; - xor_optab->libcall_suffix = '3'; - xor_optab->libcall_gen = gen_int_libfunc; - ashl_optab->libcall_basename = "ashl"; - ashl_optab->libcall_suffix = '3'; - ashl_optab->libcall_gen = gen_int_fixed_libfunc; - ssashl_optab->libcall_basename = "ssashl"; - ssashl_optab->libcall_suffix = '3'; - ssashl_optab->libcall_gen = gen_signed_fixed_libfunc; - usashl_optab->libcall_basename = "usashl"; - usashl_optab->libcall_suffix = '3'; - usashl_optab->libcall_gen = gen_unsigned_fixed_libfunc; - ashr_optab->libcall_basename = "ashr"; - ashr_optab->libcall_suffix = '3'; - ashr_optab->libcall_gen = gen_int_signed_fixed_libfunc; - lshr_optab->libcall_basename = "lshr"; - lshr_optab->libcall_suffix = '3'; - lshr_optab->libcall_gen = gen_int_unsigned_fixed_libfunc; - smin_optab->libcall_basename = "min"; - smin_optab->libcall_suffix = '3'; - smin_optab->libcall_gen = gen_int_fp_libfunc; - smax_optab->libcall_basename = "max"; - smax_optab->libcall_suffix = '3'; - smax_optab->libcall_gen = gen_int_fp_libfunc; - umin_optab->libcall_basename = "umin"; - umin_optab->libcall_suffix = '3'; - umin_optab->libcall_gen = gen_int_libfunc; - umax_optab->libcall_basename = "umax"; - umax_optab->libcall_suffix = '3'; - umax_optab->libcall_gen = gen_int_libfunc; - neg_optab->libcall_basename = "neg"; - neg_optab->libcall_suffix = '2'; - neg_optab->libcall_gen = gen_int_fp_fixed_libfunc; - ssneg_optab->libcall_basename = "ssneg"; - ssneg_optab->libcall_suffix = '2'; - ssneg_optab->libcall_gen = gen_signed_fixed_libfunc; - usneg_optab->libcall_basename = "usneg"; - usneg_optab->libcall_suffix = '2'; - usneg_optab->libcall_gen = gen_unsigned_fixed_libfunc; - negv_optab->libcall_basename = "neg"; - negv_optab->libcall_suffix = '2'; - negv_optab->libcall_gen = gen_intv_fp_libfunc; - one_cmpl_optab->libcall_basename = "one_cmpl"; - one_cmpl_optab->libcall_suffix = '2'; - one_cmpl_optab->libcall_gen = gen_int_libfunc; - ffs_optab->libcall_basename = "ffs"; - ffs_optab->libcall_suffix = '2'; - ffs_optab->libcall_gen = gen_int_libfunc; - clz_optab->libcall_basename = "clz"; - clz_optab->libcall_suffix = '2'; - clz_optab->libcall_gen = gen_int_libfunc; - ctz_optab->libcall_basename = "ctz"; - ctz_optab->libcall_suffix = '2'; - ctz_optab->libcall_gen = gen_int_libfunc; - popcount_optab->libcall_basename = "popcount"; - popcount_optab->libcall_suffix = '2'; - popcount_optab->libcall_gen = gen_int_libfunc; - parity_optab->libcall_basename = "parity"; - parity_optab->libcall_suffix = '2'; - parity_optab->libcall_gen = gen_int_libfunc; - - /* Comparison libcalls for integers MUST come in pairs, - signed/unsigned. */ - cmp_optab->libcall_basename = "cmp"; - cmp_optab->libcall_suffix = '2'; - cmp_optab->libcall_gen = gen_int_fp_fixed_libfunc; - ucmp_optab->libcall_basename = "ucmp"; - ucmp_optab->libcall_suffix = '2'; - ucmp_optab->libcall_gen = gen_int_libfunc; - - /* EQ etc are floating point only. */ - eq_optab->libcall_basename = "eq"; - eq_optab->libcall_suffix = '2'; - eq_optab->libcall_gen = gen_fp_libfunc; - ne_optab->libcall_basename = "ne"; - ne_optab->libcall_suffix = '2'; - ne_optab->libcall_gen = gen_fp_libfunc; - gt_optab->libcall_basename = "gt"; - gt_optab->libcall_suffix = '2'; - gt_optab->libcall_gen = gen_fp_libfunc; - ge_optab->libcall_basename = "ge"; - ge_optab->libcall_suffix = '2'; - ge_optab->libcall_gen = gen_fp_libfunc; - lt_optab->libcall_basename = "lt"; - lt_optab->libcall_suffix = '2'; - lt_optab->libcall_gen = gen_fp_libfunc; - le_optab->libcall_basename = "le"; - le_optab->libcall_suffix = '2'; - le_optab->libcall_gen = gen_fp_libfunc; - unord_optab->libcall_basename = "unord"; - unord_optab->libcall_suffix = '2'; - unord_optab->libcall_gen = gen_fp_libfunc; - - powi_optab->libcall_basename = "powi"; - powi_optab->libcall_suffix = '2'; - powi_optab->libcall_gen = gen_fp_libfunc; - - /* Conversions. */ - sfloat_optab->libcall_basename = "float"; - sfloat_optab->libcall_gen = gen_int_to_fp_conv_libfunc; - ufloat_optab->libcall_gen = gen_ufloat_conv_libfunc; - sfix_optab->libcall_basename = "fix"; - sfix_optab->libcall_gen = gen_fp_to_int_conv_libfunc; - ufix_optab->libcall_basename = "fixuns"; - ufix_optab->libcall_gen = gen_fp_to_int_conv_libfunc; - lrint_optab->libcall_basename = "lrint"; - lrint_optab->libcall_gen = gen_int_to_fp_nondecimal_conv_libfunc; - lround_optab->libcall_basename = "lround"; - lround_optab->libcall_gen = gen_int_to_fp_nondecimal_conv_libfunc; - lfloor_optab->libcall_basename = "lfloor"; - lfloor_optab->libcall_gen = gen_int_to_fp_nondecimal_conv_libfunc; - lceil_optab->libcall_basename = "lceil"; - lceil_optab->libcall_gen = gen_int_to_fp_nondecimal_conv_libfunc; - - /* trunc_optab is also used for FLOAT_EXTEND. */ - sext_optab->libcall_basename = "extend"; - sext_optab->libcall_gen = gen_extend_conv_libfunc; - trunc_optab->libcall_basename = "trunc"; - trunc_optab->libcall_gen = gen_trunc_conv_libfunc; - - /* Conversions for fixed-point modes and other modes. */ - fract_optab->libcall_basename = "fract"; - fract_optab->libcall_gen = gen_fract_conv_libfunc; - satfract_optab->libcall_basename = "satfract"; - satfract_optab->libcall_gen = gen_satfract_conv_libfunc; - fractuns_optab->libcall_basename = "fractuns"; - fractuns_optab->libcall_gen = gen_fractuns_conv_libfunc; - satfractuns_optab->libcall_basename = "satfractuns"; - satfractuns_optab->libcall_gen = gen_satfractuns_conv_libfunc; - - /* The ffs function operates on `int'. Fall back on it if we do not - have a libgcc2 function for that width. */ - if (INT_TYPE_SIZE < BITS_PER_WORD) - set_optab_libfunc (ffs_optab, mode_for_size (INT_TYPE_SIZE, MODE_INT, 0), - "ffs"); - - /* Explicitly initialize the bswap libfuncs since we need them to be - valid for things other than word_mode. */ - set_optab_libfunc (bswap_optab, SImode, "__bswapsi2"); - set_optab_libfunc (bswap_optab, DImode, "__bswapdi2"); - - /* Use cabs for double complex abs, since systems generally have cabs. - Don't define any libcall for float complex, so that cabs will be used. */ - if (complex_double_type_node) - set_optab_libfunc (abs_optab, TYPE_MODE (complex_double_type_node), "cabs"); - - abort_libfunc = init_one_libfunc ("abort"); - memcpy_libfunc = init_one_libfunc ("memcpy"); - memmove_libfunc = init_one_libfunc ("memmove"); - memcmp_libfunc = init_one_libfunc ("memcmp"); - memset_libfunc = init_one_libfunc ("memset"); - setbits_libfunc = init_one_libfunc ("__setbits"); - -#ifndef DONT_USE_BUILTIN_SETJMP - setjmp_libfunc = init_one_libfunc ("__builtin_setjmp"); - longjmp_libfunc = init_one_libfunc ("__builtin_longjmp"); -#else - setjmp_libfunc = init_one_libfunc ("setjmp"); - longjmp_libfunc = init_one_libfunc ("longjmp"); -#endif - unwind_sjlj_register_libfunc = init_one_libfunc ("_Unwind_SjLj_Register"); - unwind_sjlj_unregister_libfunc - = init_one_libfunc ("_Unwind_SjLj_Unregister"); - - /* For function entry/exit instrumentation. */ - profile_function_entry_libfunc - = init_one_libfunc ("__cyg_profile_func_enter"); - profile_function_exit_libfunc - = init_one_libfunc ("__cyg_profile_func_exit"); - - gcov_flush_libfunc = init_one_libfunc ("__gcov_flush"); - - /* Allow the target to add more libcalls or rename some, etc. */ - targetm.init_libfuncs (); + return (code_to_optab (code) + && (optab_handler (code_to_optab (code), mode) + != CODE_FOR_nothing)); } /* Print information about the current contents of the optabs on @@ -6577,45 +5163,35 @@ DEBUG_FUNCTION void debug_optab_libfuncs (void) { - int i; - int j; - int k; + int i, j, k; /* Dump the arithmetic optabs. */ - for (i = 0; i != (int) OTI_MAX; i++) + for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i) for (j = 0; j < NUM_MACHINE_MODES; ++j) { - optab o; - rtx l; - - o = &optab_table[i]; - l = optab_libfunc (o, (enum machine_mode) j); + rtx l = optab_libfunc ((optab) i, (machine_mode) j); if (l) { gcc_assert (GET_CODE (l) == SYMBOL_REF); fprintf (stderr, "%s\t%s:\t%s\n", - GET_RTX_NAME (o->code), + GET_RTX_NAME (optab_to_code ((optab) i)), GET_MODE_NAME (j), XSTR (l, 0)); } } /* Dump the conversion optabs. */ - for (i = 0; i < (int) COI_MAX; ++i) + for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i) for (j = 0; j < NUM_MACHINE_MODES; ++j) for (k = 0; k < NUM_MACHINE_MODES; ++k) { - convert_optab o; - rtx l; - - o = &convert_optab_table[i]; - l = convert_optab_libfunc (o, (enum machine_mode) j, - (enum machine_mode) k); + rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j, + (machine_mode) k); if (l) { gcc_assert (GET_CODE (l) == SYMBOL_REF); fprintf (stderr, "%s\t%s\t%s:\t%s\n", - GET_RTX_NAME (o->code), + GET_RTX_NAME (optab_to_code ((optab) i)), GET_MODE_NAME (j), GET_MODE_NAME (k), XSTR (l, 0)); @@ -6623,16 +5199,15 @@ } } - /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition CODE. Return 0 on failure. */ -rtx +rtx_insn * gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode) { - enum machine_mode mode = GET_MODE (op1); + machine_mode mode = GET_MODE (op1); enum insn_code icode; - rtx insn; + rtx_insn *insn; rtx trap_rtx; if (mode == VOIDmode) @@ -6643,8 +5218,7 @@ return 0; /* Some targets only accept a zero trap code. */ - if (insn_data[icode].operand[3].predicate - && !insn_data[icode].operand[3].predicate (tcode, VOIDmode)) + if (!insn_operand_matches (icode, 3, tcode)) return 0; do_pending_stack_adjust (); @@ -6652,7 +5226,7 @@ prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT, &trap_rtx, &mode); if (!trap_rtx) - insn = NULL_RTX; + insn = NULL; else insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1), tcode); @@ -6673,7 +5247,7 @@ /* Return rtx code for TCODE. Use UNSIGNEDP to select signed or unsigned operation code. */ -static enum rtx_code +enum rtx_code get_rtx_code (enum tree_code tcode, bool unsignedp) { enum rtx_code code; @@ -6723,70 +5297,322 @@ code = LTGT; break; + case BIT_AND_EXPR: + code = AND; + break; + + case BIT_IOR_EXPR: + code = IOR; + break; + default: gcc_unreachable (); } return code; } -/* Return comparison rtx for COND. Use UNSIGNEDP to select signed or - unsigned operators. Do not generate compare instruction. */ +/* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to + select signed or unsigned operators. OPNO holds the index of the + first comparison operand for insn ICODE. Do not generate the + compare instruction itself. */ static rtx -vector_compare_rtx (tree cond, bool unsignedp, enum insn_code icode) +vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode, + tree t_op0, tree t_op1, bool unsignedp, + enum insn_code icode, unsigned int opno) { - enum rtx_code rcode; - tree t_op0, t_op1; + struct expand_operand ops[2]; rtx rtx_op0, rtx_op1; - - /* This is unlikely. While generating VEC_COND_EXPR, auto vectorizer - ensures that condition is a relational operation. */ - gcc_assert (COMPARISON_CLASS_P (cond)); - - rcode = get_rtx_code (TREE_CODE (cond), unsignedp); - t_op0 = TREE_OPERAND (cond, 0); - t_op1 = TREE_OPERAND (cond, 1); - - /* Expand operands. */ + machine_mode m0, m1; + enum rtx_code rcode = get_rtx_code (tcode, unsignedp); + + gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison); + + /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t + has mode DImode, this can produce a constant RTX of mode VOIDmode; in such + cases, use the original mode. */ rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)), EXPAND_STACK_PARM); + m0 = GET_MODE (rtx_op0); + if (m0 == VOIDmode) + m0 = TYPE_MODE (TREE_TYPE (t_op0)); + rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)), EXPAND_STACK_PARM); - - if (!insn_data[icode].operand[4].predicate (rtx_op0, GET_MODE (rtx_op0)) - && GET_MODE (rtx_op0) != VOIDmode) - rtx_op0 = force_reg (GET_MODE (rtx_op0), rtx_op0); - - if (!insn_data[icode].operand[5].predicate (rtx_op1, GET_MODE (rtx_op1)) - && GET_MODE (rtx_op1) != VOIDmode) - rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); - - return gen_rtx_fmt_ee (rcode, VOIDmode, rtx_op0, rtx_op1); + m1 = GET_MODE (rtx_op1); + if (m1 == VOIDmode) + m1 = TYPE_MODE (TREE_TYPE (t_op1)); + + create_input_operand (&ops[0], rtx_op0, m0); + create_input_operand (&ops[1], rtx_op1, m1); + if (!maybe_legitimize_operands (icode, opno, 2, ops)) + gcc_unreachable (); + return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value); +} + +/* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first + vec_perm operand, assuming the second operand is a constant vector of zeroes. + Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a + shift. */ +static rtx +shift_amt_for_vec_perm_mask (rtx sel) +{ + unsigned int i, first, nelt = GET_MODE_NUNITS (GET_MODE (sel)); + unsigned int bitsize = GET_MODE_UNIT_BITSIZE (GET_MODE (sel)); + + if (GET_CODE (sel) != CONST_VECTOR) + return NULL_RTX; + + first = INTVAL (CONST_VECTOR_ELT (sel, 0)); + if (first >= nelt) + return NULL_RTX; + for (i = 1; i < nelt; i++) + { + int idx = INTVAL (CONST_VECTOR_ELT (sel, i)); + unsigned int expected = i + first; + /* Indices into the second vector are all equivalent. */ + if (idx < 0 || (MIN (nelt, (unsigned) idx) != MIN (nelt, expected))) + return NULL_RTX; + } + + return GEN_INT (first * bitsize); } -/* Return insn code for TYPE, the type of a VEC_COND_EXPR. */ - -static inline enum insn_code -get_vcond_icode (tree type, enum machine_mode mode) +/* A subroutine of expand_vec_perm for expanding one vec_perm insn. */ + +static rtx +expand_vec_perm_1 (enum insn_code icode, rtx target, + rtx v0, rtx v1, rtx sel) +{ + machine_mode tmode = GET_MODE (target); + machine_mode smode = GET_MODE (sel); + struct expand_operand ops[4]; + + create_output_operand (&ops[0], target, tmode); + create_input_operand (&ops[3], sel, smode); + + /* Make an effort to preserve v0 == v1. The target expander is able to + rely on this to determine if we're permuting a single input operand. */ + if (rtx_equal_p (v0, v1)) + { + if (!insn_operand_matches (icode, 1, v0)) + v0 = force_reg (tmode, v0); + gcc_checking_assert (insn_operand_matches (icode, 1, v0)); + gcc_checking_assert (insn_operand_matches (icode, 2, v0)); + + create_fixed_operand (&ops[1], v0); + create_fixed_operand (&ops[2], v0); + } + else + { + create_input_operand (&ops[1], v0, tmode); + create_input_operand (&ops[2], v1, tmode); + } + + if (maybe_expand_insn (icode, 4, ops)) + return ops[0].value; + return NULL_RTX; +} + +/* Generate instructions for vec_perm optab given its mode + and three operands. */ + +rtx +expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) { - enum insn_code icode = CODE_FOR_nothing; - - if (TYPE_UNSIGNED (type)) - icode = direct_optab_handler (vcondu_optab, mode); - else - icode = direct_optab_handler (vcond_optab, mode); - return icode; + enum insn_code icode; + machine_mode qimode; + unsigned int i, w, e, u; + rtx tmp, sel_qi = NULL; + rtvec vec; + + if (!target || GET_MODE (target) != mode) + target = gen_reg_rtx (mode); + + w = GET_MODE_SIZE (mode); + e = GET_MODE_NUNITS (mode); + u = GET_MODE_UNIT_SIZE (mode); + + /* Set QIMODE to a different vector mode with byte elements. + If no such mode, or if MODE already has byte elements, use VOIDmode. */ + if (GET_MODE_INNER (mode) == QImode + || !mode_for_vector (QImode, w).exists (&qimode) + || !VECTOR_MODE_P (qimode)) + qimode = VOIDmode; + + /* If the input is a constant, expand it specially. */ + gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT); + if (GET_CODE (sel) == CONST_VECTOR) + { + /* See if this can be handled with a vec_shr. We only do this if the + second vector is all zeroes. */ + enum insn_code shift_code = optab_handler (vec_shr_optab, mode); + enum insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode) + ? optab_handler (vec_shr_optab, qimode) + : CODE_FOR_nothing); + rtx shift_amt = NULL_RTX; + if (v1 == CONST0_RTX (GET_MODE (v1)) + && (shift_code != CODE_FOR_nothing + || shift_code_qi != CODE_FOR_nothing)) + { + shift_amt = shift_amt_for_vec_perm_mask (sel); + if (shift_amt) + { + struct expand_operand ops[3]; + if (shift_code != CODE_FOR_nothing) + { + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], v0, mode); + create_convert_operand_from_type (&ops[2], shift_amt, + sizetype); + if (maybe_expand_insn (shift_code, 3, ops)) + return ops[0].value; + } + if (shift_code_qi != CODE_FOR_nothing) + { + tmp = gen_reg_rtx (qimode); + create_output_operand (&ops[0], tmp, qimode); + create_input_operand (&ops[1], gen_lowpart (qimode, v0), + qimode); + create_convert_operand_from_type (&ops[2], shift_amt, + sizetype); + if (maybe_expand_insn (shift_code_qi, 3, ops)) + return gen_lowpart (mode, ops[0].value); + } + } + } + + icode = direct_optab_handler (vec_perm_const_optab, mode); + if (icode != CODE_FOR_nothing) + { + tmp = expand_vec_perm_1 (icode, target, v0, v1, sel); + if (tmp) + return tmp; + } + + /* Fall back to a constant byte-based permutation. */ + if (qimode != VOIDmode) + { + vec = rtvec_alloc (w); + for (i = 0; i < e; ++i) + { + unsigned int j, this_e; + + this_e = INTVAL (CONST_VECTOR_ELT (sel, i)); + this_e &= 2 * e - 1; + this_e *= u; + + for (j = 0; j < u; ++j) + RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j); + } + sel_qi = gen_rtx_CONST_VECTOR (qimode, vec); + + icode = direct_optab_handler (vec_perm_const_optab, qimode); + if (icode != CODE_FOR_nothing) + { + tmp = mode != qimode ? gen_reg_rtx (qimode) : target; + tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0), + gen_lowpart (qimode, v1), sel_qi); + if (tmp) + return gen_lowpart (mode, tmp); + } + } + } + + /* Otherwise expand as a fully variable permuation. */ + icode = direct_optab_handler (vec_perm_optab, mode); + if (icode != CODE_FOR_nothing) + { + tmp = expand_vec_perm_1 (icode, target, v0, v1, sel); + if (tmp) + return tmp; + } + + /* As a special case to aid several targets, lower the element-based + permutation to a byte-based permutation and try again. */ + if (qimode == VOIDmode) + return NULL_RTX; + icode = direct_optab_handler (vec_perm_optab, qimode); + if (icode == CODE_FOR_nothing) + return NULL_RTX; + + if (sel_qi == NULL) + { + /* Multiply each element by its byte size. */ + machine_mode selmode = GET_MODE (sel); + if (u == 2) + sel = expand_simple_binop (selmode, PLUS, sel, sel, + NULL, 0, OPTAB_DIRECT); + else + sel = expand_simple_binop (selmode, ASHIFT, sel, + GEN_INT (exact_log2 (u)), + NULL, 0, OPTAB_DIRECT); + gcc_assert (sel != NULL); + + /* Broadcast the low byte each element into each of its bytes. */ + vec = rtvec_alloc (w); + for (i = 0; i < w; ++i) + { + int this_e = i / u * u; + if (BYTES_BIG_ENDIAN) + this_e += u - 1; + RTVEC_ELT (vec, i) = GEN_INT (this_e); + } + tmp = gen_rtx_CONST_VECTOR (qimode, vec); + sel = gen_lowpart (qimode, sel); + sel = expand_vec_perm (qimode, sel, sel, tmp, NULL); + gcc_assert (sel != NULL); + + /* Add the byte offset to each byte element. */ + /* Note that the definition of the indicies here is memory ordering, + so there should be no difference between big and little endian. */ + vec = rtvec_alloc (w); + for (i = 0; i < w; ++i) + RTVEC_ELT (vec, i) = GEN_INT (i % u); + tmp = gen_rtx_CONST_VECTOR (qimode, vec); + sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp, + sel, 0, OPTAB_DIRECT); + gcc_assert (sel_qi != NULL); + } + + tmp = mode != qimode ? gen_reg_rtx (qimode) : target; + tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0), + gen_lowpart (qimode, v1), sel_qi); + if (tmp) + tmp = gen_lowpart (mode, tmp); + return tmp; } -/* Return TRUE iff, appropriate vector insns are available - for vector cond expr with type TYPE in VMODE mode. */ - -bool -expand_vec_cond_expr_p (tree type, enum machine_mode vmode) +/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its + three operands. */ + +rtx +expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, + rtx target) { - if (get_vcond_icode (type, vmode) == CODE_FOR_nothing) - return false; - return true; + struct expand_operand ops[4]; + machine_mode mode = TYPE_MODE (vec_cond_type); + machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); + enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); + rtx mask, rtx_op1, rtx_op2; + + if (icode == CODE_FOR_nothing) + return 0; + + mask = expand_normal (op0); + rtx_op1 = expand_normal (op1); + rtx_op2 = expand_normal (op2); + + mask = force_reg (mask_mode, mask); + rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); + + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], rtx_op1, mode); + create_input_operand (&ops[2], rtx_op2, mode); + create_input_operand (&ops[3], mask, mask_mode); + expand_insn (icode, 4, ops); + + return ops[0].value; } /* Generate insns for a VEC_COND_EXPR, given its TYPE and its @@ -6796,91 +5622,181 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, rtx target) { + struct expand_operand ops[6]; enum insn_code icode; - rtx comparison, rtx_op1, rtx_op2, cc_op0, cc_op1; - enum machine_mode mode = TYPE_MODE (vec_cond_type); - bool unsignedp = TYPE_UNSIGNED (vec_cond_type); - - icode = get_vcond_icode (vec_cond_type, mode); + rtx comparison, rtx_op1, rtx_op2; + machine_mode mode = TYPE_MODE (vec_cond_type); + machine_mode cmp_op_mode; + bool unsignedp; + tree op0a, op0b; + enum tree_code tcode; + + if (COMPARISON_CLASS_P (op0)) + { + op0a = TREE_OPERAND (op0, 0); + op0b = TREE_OPERAND (op0, 1); + tcode = TREE_CODE (op0); + } + else + { + gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); + if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) + != CODE_FOR_nothing) + return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, + op2, target); + /* Fake op0 < 0. */ + else + { + gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) + == MODE_VECTOR_INT); + op0a = op0; + op0b = build_zero_cst (TREE_TYPE (op0)); + tcode = LT_EXPR; + } + } + cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); + unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); + + + gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode) + && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode)); + + icode = get_vcond_icode (mode, cmp_op_mode, unsignedp); if (icode == CODE_FOR_nothing) - return 0; - - if (!target || !insn_data[icode].operand[0].predicate (target, mode)) - target = gen_reg_rtx (mode); - - /* Get comparison rtx. First expand both cond expr operands. */ - comparison = vector_compare_rtx (op0, - unsignedp, icode); - cc_op0 = XEXP (comparison, 0); - cc_op1 = XEXP (comparison, 1); - /* Expand both operands and force them in reg, if required. */ + { + if (tcode == EQ_EXPR || tcode == NE_EXPR) + icode = get_vcond_eq_icode (mode, cmp_op_mode); + if (icode == CODE_FOR_nothing) + return 0; + } + + comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, + icode, 4); rtx_op1 = expand_normal (op1); - if (!insn_data[icode].operand[1].predicate (rtx_op1, mode) - && mode != VOIDmode) - rtx_op1 = force_reg (mode, rtx_op1); - rtx_op2 = expand_normal (op2); - if (!insn_data[icode].operand[2].predicate (rtx_op2, mode) - && mode != VOIDmode) - rtx_op2 = force_reg (mode, rtx_op2); - - /* Emit instruction! */ - emit_insn (GEN_FCN (icode) (target, rtx_op1, rtx_op2, - comparison, cc_op0, cc_op1)); - - return target; + + create_output_operand (&ops[0], target, mode); + create_input_operand (&ops[1], rtx_op1, mode); + create_input_operand (&ops[2], rtx_op2, mode); + create_fixed_operand (&ops[3], comparison); + create_fixed_operand (&ops[4], XEXP (comparison, 0)); + create_fixed_operand (&ops[5], XEXP (comparison, 1)); + expand_insn (icode, 6, ops); + return ops[0].value; } - -/* This is an internal subroutine of the other compare_and_swap expanders. - MEM, OLD_VAL and NEW_VAL are as you'd expect for a compare-and-swap - operation. TARGET is an optional place to store the value result of - the operation. ICODE is the particular instruction to expand. Return - the result of the operation. */ - -static rtx -expand_val_compare_and_swap_1 (rtx mem, rtx old_val, rtx new_val, - rtx target, enum insn_code icode) -{ - enum machine_mode mode = GET_MODE (mem); - rtx insn; - - if (!target || !insn_data[icode].operand[0].predicate (target, mode)) - target = gen_reg_rtx (mode); - - if (GET_MODE (old_val) != VOIDmode && GET_MODE (old_val) != mode) - old_val = convert_modes (mode, GET_MODE (old_val), old_val, 1); - if (!insn_data[icode].operand[2].predicate (old_val, mode)) - old_val = force_reg (mode, old_val); - - if (GET_MODE (new_val) != VOIDmode && GET_MODE (new_val) != mode) - new_val = convert_modes (mode, GET_MODE (new_val), new_val, 1); - if (!insn_data[icode].operand[3].predicate (new_val, mode)) - new_val = force_reg (mode, new_val); - - insn = GEN_FCN (icode) (target, mem, old_val, new_val); - if (insn == NULL_RTX) - return NULL_RTX; - emit_insn (insn); - - return target; -} - -/* Expand a compare-and-swap operation and return its value. */ +/* Generate insns for a vector comparison into a mask. */ rtx -expand_val_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target) +expand_vec_cmp_expr (tree type, tree exp, rtx target) +{ + struct expand_operand ops[4]; + enum insn_code icode; + rtx comparison; + machine_mode mask_mode = TYPE_MODE (type); + machine_mode vmode; + bool unsignedp; + tree op0a, op0b; + enum tree_code tcode; + + op0a = TREE_OPERAND (exp, 0); + op0b = TREE_OPERAND (exp, 1); + tcode = TREE_CODE (exp); + + unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); + vmode = TYPE_MODE (TREE_TYPE (op0a)); + + icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp); + if (icode == CODE_FOR_nothing) + { + if (tcode == EQ_EXPR || tcode == NE_EXPR) + icode = get_vec_cmp_eq_icode (vmode, mask_mode); + if (icode == CODE_FOR_nothing) + return 0; + } + + comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b, + unsignedp, icode, 2); + create_output_operand (&ops[0], target, mask_mode); + create_fixed_operand (&ops[1], comparison); + create_fixed_operand (&ops[2], XEXP (comparison, 0)); + create_fixed_operand (&ops[3], XEXP (comparison, 1)); + expand_insn (icode, 4, ops); + return ops[0].value; +} + +/* Expand a highpart multiply. */ + +rtx +expand_mult_highpart (machine_mode mode, rtx op0, rtx op1, + rtx target, bool uns_p) { - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode - = direct_optab_handler (sync_compare_and_swap_optab, mode); - - if (icode == CODE_FOR_nothing) - return NULL_RTX; - - return expand_val_compare_and_swap_1 (mem, old_val, new_val, target, icode); + struct expand_operand eops[3]; + enum insn_code icode; + int method, i, nunits; + machine_mode wmode; + rtx m1, m2, perm; + optab tab1, tab2; + rtvec v; + + method = can_mult_highpart_p (mode, uns_p); + switch (method) + { + case 0: + return NULL_RTX; + case 1: + tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab; + return expand_binop (mode, tab1, op0, op1, target, uns_p, + OPTAB_LIB_WIDEN); + case 2: + tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab; + tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab; + break; + case 3: + tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab; + tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab; + if (BYTES_BIG_ENDIAN) + std::swap (tab1, tab2); + break; + default: + gcc_unreachable (); + } + + icode = optab_handler (tab1, mode); + nunits = GET_MODE_NUNITS (mode); + wmode = insn_data[icode].operand[0].mode; + gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits); + gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode)); + + create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode); + create_input_operand (&eops[1], op0, mode); + create_input_operand (&eops[2], op1, mode); + expand_insn (icode, 3, eops); + m1 = gen_lowpart (mode, eops[0].value); + + create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode); + create_input_operand (&eops[1], op0, mode); + create_input_operand (&eops[2], op1, mode); + expand_insn (optab_handler (tab2, mode), 3, eops); + m2 = gen_lowpart (mode, eops[0].value); + + v = rtvec_alloc (nunits); + if (method == 2) + { + for (i = 0; i < nunits; ++i) + RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1) + + ((i & 1) ? nunits : 0)); + } + else + { + for (i = 0; i < nunits; ++i) + RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1)); + } + perm = gen_rtx_CONST_VECTOR (mode, v); + + return expand_vec_perm (mode, m1, m2, perm, target); } - + /* Helper function to find the MODE_CC set in a sync_compare_and_swap pattern. */ @@ -6896,58 +5812,6 @@ } } -/* Expand a compare-and-swap operation and store true into the result if - the operation was successful and false otherwise. Return the result. - Unlike other routines, TARGET is not optional. */ - -rtx -expand_bool_compare_and_swap (rtx mem, rtx old_val, rtx new_val, rtx target) -{ - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode; - rtx subtarget, seq, cc_reg; - - /* If the target supports a compare-and-swap pattern that simultaneously - sets some flag for success, then use it. Otherwise use the regular - compare-and-swap and follow that immediately with a compare insn. */ - icode = direct_optab_handler (sync_compare_and_swap_optab, mode); - if (icode == CODE_FOR_nothing) - return NULL_RTX; - - do_pending_stack_adjust (); - do - { - start_sequence (); - subtarget = expand_val_compare_and_swap_1 (mem, old_val, new_val, - NULL_RTX, icode); - cc_reg = NULL_RTX; - if (subtarget == NULL_RTX) - { - end_sequence (); - return NULL_RTX; - } - - if (have_insn_for (COMPARE, CCmode)) - note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); - seq = get_insns (); - end_sequence (); - - /* We might be comparing against an old value. Try again. :-( */ - if (!cc_reg && MEM_P (old_val)) - { - seq = NULL_RTX; - old_val = force_reg (mode, old_val); - } - } - while (!seq); - - emit_insn (seq); - if (cc_reg) - return emit_store_flag_force (target, EQ, cc_reg, const0_rtx, VOIDmode, 0, 1); - else - return emit_store_flag_force (target, EQ, subtarget, old_val, VOIDmode, 1, 1); -} - /* This is a helper function for the other atomic operations. This function emits a loop that contains SEQ that iterates until a compare-and-swap operation at the end succeeds. MEM is the memory to be modified. SEQ is @@ -6960,9 +5824,9 @@ static bool expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) { - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode; - rtx label, cmp_reg, subtarget, cc_reg; + machine_mode mode = GET_MODE (mem); + rtx_code_label *label; + rtx cmp_reg, success, oldval; /* The loop we want to generate looks like @@ -6970,8 +5834,8 @@ label: old_reg = cmp_reg; seq; - cmp_reg = compare-and-swap(mem, old_reg, new_reg) - if (cmp_reg != old_reg) + (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg) + if (success) goto label; Note that we only do the plain load from memory once. Subsequent @@ -6986,343 +5850,121 @@ if (seq) emit_insn (seq); - /* If the target supports a compare-and-swap pattern that simultaneously - sets some flag for success, then use it. Otherwise use the regular - compare-and-swap and follow that immediately with a compare insn. */ - icode = direct_optab_handler (sync_compare_and_swap_optab, mode); - if (icode == CODE_FOR_nothing) - return false; - - subtarget = expand_val_compare_and_swap_1 (mem, old_reg, new_reg, - cmp_reg, icode); - if (subtarget == NULL_RTX) + success = NULL_RTX; + oldval = cmp_reg; + if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg, + new_reg, false, MEMMODEL_SYNC_SEQ_CST, + MEMMODEL_RELAXED)) return false; - cc_reg = NULL_RTX; - if (have_insn_for (COMPARE, CCmode)) - note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); - if (cc_reg) - { - cmp_reg = cc_reg; - old_reg = const0_rtx; - } - else - { - if (subtarget != cmp_reg) - emit_move_insn (cmp_reg, subtarget); - } - - /* ??? Mark this jump predicted not taken? */ - emit_cmp_and_jump_insns (cmp_reg, old_reg, NE, const0_rtx, GET_MODE (cmp_reg), 1, - label); + if (oldval != cmp_reg) + emit_move_insn (cmp_reg, oldval); + + /* Mark this jump predicted not taken. */ + emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx, + GET_MODE (success), 1, label, + profile_probability::guessed_never ()); return true; } -/* This function generates the atomic operation MEM CODE= VAL. In this - case, we do not care about any resulting value. Returns NULL if we - cannot generate the operation. */ - -rtx -expand_sync_operation (rtx mem, rtx val, enum rtx_code code) + +/* This function tries to emit an atomic_exchange intruction. VAL is written + to *MEM using memory model MODEL. The previous contents of *MEM are returned, + using TARGET if possible. */ + +static rtx +maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model) { - enum machine_mode mode = GET_MODE (mem); + machine_mode mode = GET_MODE (mem); enum insn_code icode; - rtx insn; - - /* Look to see if the target supports the operation directly. */ - switch (code) - { - case PLUS: - icode = direct_optab_handler (sync_add_optab, mode); - break; - case IOR: - icode = direct_optab_handler (sync_ior_optab, mode); - break; - case XOR: - icode = direct_optab_handler (sync_xor_optab, mode); - break; - case AND: - icode = direct_optab_handler (sync_and_optab, mode); - break; - case NOT: - icode = direct_optab_handler (sync_nand_optab, mode); - break; - - case MINUS: - icode = direct_optab_handler (sync_sub_optab, mode); - if (icode == CODE_FOR_nothing || CONST_INT_P (val)) - { - icode = direct_optab_handler (sync_add_optab, mode); - if (icode != CODE_FOR_nothing) - { - val = expand_simple_unop (mode, NEG, val, NULL_RTX, 1); - code = PLUS; - } - } - break; - - default: - gcc_unreachable (); - } - - /* Generate the direct operation, if present. */ + + /* If the target supports the exchange directly, great. */ + icode = direct_optab_handler (atomic_exchange_optab, mode); if (icode != CODE_FOR_nothing) { - if (GET_MODE (val) != VOIDmode && GET_MODE (val) != mode) - val = convert_modes (mode, GET_MODE (val), val, 1); - if (!insn_data[icode].operand[1].predicate (val, mode)) - val = force_reg (mode, val); - - insn = GEN_FCN (icode) (mem, val); - if (insn) - { - emit_insn (insn); - return const0_rtx; - } - } - - /* Failing that, generate a compare-and-swap loop in which we perform the - operation with normal arithmetic instructions. */ - if (direct_optab_handler (sync_compare_and_swap_optab, mode) - != CODE_FOR_nothing) - { - rtx t0 = gen_reg_rtx (mode), t1; - - start_sequence (); - - t1 = t0; - if (code == NOT) - { - t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX, - true, OPTAB_LIB_WIDEN); - t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true); - } - else - t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, - true, OPTAB_LIB_WIDEN); - insn = get_insns (); - end_sequence (); - - if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn)) - return const0_rtx; + struct expand_operand ops[4]; + + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], mem); + create_input_operand (&ops[2], val, mode); + create_integer_operand (&ops[3], model); + if (maybe_expand_insn (icode, 4, ops)) + return ops[0].value; } return NULL_RTX; } -/* This function generates the atomic operation MEM CODE= VAL. In this - case, we do care about the resulting value: if AFTER is true then - return the value MEM holds after the operation, if AFTER is false - then return the value MEM holds before the operation. TARGET is an - optional place for the result value to be stored. */ - -rtx -expand_sync_fetch_operation (rtx mem, rtx val, enum rtx_code code, - bool after, rtx target) +/* This function tries to implement an atomic exchange operation using + __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL. + The previous contents of *MEM are returned, using TARGET if possible. + Since this instructionn is an acquire barrier only, stronger memory + models may require additional barriers to be emitted. */ + +static rtx +maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val, + enum memmodel model) { - enum machine_mode mode = GET_MODE (mem); - enum insn_code old_code, new_code, icode; - bool compensate; - rtx insn; - - /* Look to see if the target supports the operation directly. */ - switch (code) - { - case PLUS: - old_code = direct_optab_handler (sync_old_add_optab, mode); - new_code = direct_optab_handler (sync_new_add_optab, mode); - break; - case IOR: - old_code = direct_optab_handler (sync_old_ior_optab, mode); - new_code = direct_optab_handler (sync_new_ior_optab, mode); - break; - case XOR: - old_code = direct_optab_handler (sync_old_xor_optab, mode); - new_code = direct_optab_handler (sync_new_xor_optab, mode); - break; - case AND: - old_code = direct_optab_handler (sync_old_and_optab, mode); - new_code = direct_optab_handler (sync_new_and_optab, mode); - break; - case NOT: - old_code = direct_optab_handler (sync_old_nand_optab, mode); - new_code = direct_optab_handler (sync_new_nand_optab, mode); - break; - - case MINUS: - old_code = direct_optab_handler (sync_old_sub_optab, mode); - new_code = direct_optab_handler (sync_new_sub_optab, mode); - if ((old_code == CODE_FOR_nothing && new_code == CODE_FOR_nothing) - || CONST_INT_P (val)) - { - old_code = direct_optab_handler (sync_old_add_optab, mode); - new_code = direct_optab_handler (sync_new_add_optab, mode); - if (old_code != CODE_FOR_nothing || new_code != CODE_FOR_nothing) - { - val = expand_simple_unop (mode, NEG, val, NULL_RTX, 1); - code = PLUS; - } - } - break; - - default: - gcc_unreachable (); - } - - /* If the target does supports the proper new/old operation, great. But - if we only support the opposite old/new operation, check to see if we - can compensate. In the case in which the old value is supported, then - we can always perform the operation again with normal arithmetic. In - the case in which the new value is supported, then we can only handle - this in the case the operation is reversible. */ - compensate = false; - if (after) - { - icode = new_code; - if (icode == CODE_FOR_nothing) - { - icode = old_code; - if (icode != CODE_FOR_nothing) - compensate = true; - } - } - else - { - icode = old_code; - if (icode == CODE_FOR_nothing - && (code == PLUS || code == MINUS || code == XOR)) + machine_mode mode = GET_MODE (mem); + enum insn_code icode; + rtx_insn *last_insn = get_last_insn (); + + icode = optab_handler (sync_lock_test_and_set_optab, mode); + + /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern + exists, and the memory model is stronger than acquire, add a release + barrier before the instruction. */ + + if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model)) + expand_mem_thread_fence (model); + + if (icode != CODE_FOR_nothing) + { + struct expand_operand ops[3]; + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], mem); + create_input_operand (&ops[2], val, mode); + if (maybe_expand_insn (icode, 3, ops)) + return ops[0].value; + } + + /* If an external test-and-set libcall is provided, use that instead of + any external compare-and-swap that we might get from the compare-and- + swap-loop expansion later. */ + if (!can_compare_and_swap_p (mode, false)) + { + rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode); + if (libfunc != NULL) { - icode = new_code; - if (icode != CODE_FOR_nothing) - compensate = true; + rtx addr; + + addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); + return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL, + mode, addr, ptr_mode, + val, mode); } } - /* If we found something supported, great. */ - if (icode != CODE_FOR_nothing) - { - if (!target || !insn_data[icode].operand[0].predicate (target, mode)) - target = gen_reg_rtx (mode); - - if (GET_MODE (val) != VOIDmode && GET_MODE (val) != mode) - val = convert_modes (mode, GET_MODE (val), val, 1); - if (!insn_data[icode].operand[2].predicate (val, mode)) - val = force_reg (mode, val); - - insn = GEN_FCN (icode) (target, mem, val); - if (insn) - { - emit_insn (insn); - - /* If we need to compensate for using an operation with the - wrong return value, do so now. */ - if (compensate) - { - if (!after) - { - if (code == PLUS) - code = MINUS; - else if (code == MINUS) - code = PLUS; - } - - if (code == NOT) - { - target = expand_simple_binop (mode, AND, target, val, - NULL_RTX, true, - OPTAB_LIB_WIDEN); - target = expand_simple_unop (mode, code, target, - NULL_RTX, true); - } - else - target = expand_simple_binop (mode, code, target, val, - NULL_RTX, true, - OPTAB_LIB_WIDEN); - } - - return target; - } - } - - /* Failing that, generate a compare-and-swap loop in which we perform the - operation with normal arithmetic instructions. */ - if (direct_optab_handler (sync_compare_and_swap_optab, mode) - != CODE_FOR_nothing) - { - rtx t0 = gen_reg_rtx (mode), t1; - - if (!target || !register_operand (target, mode)) - target = gen_reg_rtx (mode); - - start_sequence (); - - if (!after) - emit_move_insn (target, t0); - t1 = t0; - if (code == NOT) - { - t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX, - true, OPTAB_LIB_WIDEN); - t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true); - } - else - t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, - true, OPTAB_LIB_WIDEN); - if (after) - emit_move_insn (target, t1); - - insn = get_insns (); - end_sequence (); - - if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn)) - return target; - } - + /* If the test_and_set can't be emitted, eliminate any barrier that might + have been emitted. */ + delete_insns_since (last_insn); return NULL_RTX; } -/* This function expands a test-and-set operation. Ideally we atomically - store VAL in MEM and return the previous value in MEM. Some targets - may not support this operation and only support VAL with the constant 1; - in this case while the return value will be 0/1, but the exact value - stored in MEM is target defined. TARGET is an option place to stick - the return value. */ - -rtx -expand_sync_lock_test_and_set (rtx mem, rtx val, rtx target) +/* This function tries to implement an atomic exchange operation using a + compare_and_swap loop. VAL is written to *MEM. The previous contents of + *MEM are returned, using TARGET if possible. No memory model is required + since a compare_and_swap loop is seq-cst. */ + +static rtx +maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val) { - enum machine_mode mode = GET_MODE (mem); - enum insn_code icode; - rtx insn; - - /* If the target supports the test-and-set directly, great. */ - icode = direct_optab_handler (sync_lock_test_and_set_optab, mode); - if (icode != CODE_FOR_nothing) - { - if (!target || !insn_data[icode].operand[0].predicate (target, mode)) - target = gen_reg_rtx (mode); - - if (GET_MODE (val) != VOIDmode && GET_MODE (val) != mode) - val = convert_modes (mode, GET_MODE (val), val, 1); - if (!insn_data[icode].operand[2].predicate (val, mode)) - val = force_reg (mode, val); - - insn = GEN_FCN (icode) (target, mem, val); - if (insn) - { - emit_insn (insn); - return target; - } - } - - /* Otherwise, use a compare-and-swap loop for the exchange. */ - if (direct_optab_handler (sync_compare_and_swap_optab, mode) - != CODE_FOR_nothing) + machine_mode mode = GET_MODE (mem); + + if (can_compare_and_swap_p (mode, true)) { if (!target || !register_operand (target, mode)) target = gen_reg_rtx (mode); - if (GET_MODE (val) != VOIDmode && GET_MODE (val) != mode) - val = convert_modes (mode, GET_MODE (val), val, 1); if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX)) return target; } @@ -7330,4 +5972,1200 @@ return NULL_RTX; } -#include "gt-optabs.h" +/* This function tries to implement an atomic test-and-set operation + using the atomic_test_and_set instruction pattern. A boolean value + is returned from the operation, using TARGET if possible. */ + +static rtx +maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model) +{ + machine_mode pat_bool_mode; + struct expand_operand ops[3]; + + if (!targetm.have_atomic_test_and_set ()) + return NULL_RTX; + + /* While we always get QImode from __atomic_test_and_set, we get + other memory modes from __sync_lock_test_and_set. Note that we + use no endian adjustment here. This matches the 4.6 behavior + in the Sparc backend. */ + enum insn_code icode = targetm.code_for_atomic_test_and_set; + gcc_checking_assert (insn_data[icode].operand[1].mode == QImode); + if (GET_MODE (mem) != QImode) + mem = adjust_address_nv (mem, QImode, 0); + + pat_bool_mode = insn_data[icode].operand[0].mode; + create_output_operand (&ops[0], target, pat_bool_mode); + create_fixed_operand (&ops[1], mem); + create_integer_operand (&ops[2], model); + + if (maybe_expand_insn (icode, 3, ops)) + return ops[0].value; + return NULL_RTX; +} + +/* This function expands the legacy _sync_lock test_and_set operation which is + generally an atomic exchange. Some limited targets only allow the + constant 1 to be stored. This is an ACQUIRE operation. + + TARGET is an optional place to stick the return value. + MEM is where VAL is stored. */ + +rtx +expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val) +{ + rtx ret; + + /* Try an atomic_exchange first. */ + ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE); + if (ret) + return ret; + + ret = maybe_emit_sync_lock_test_and_set (target, mem, val, + MEMMODEL_SYNC_ACQUIRE); + if (ret) + return ret; + + ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val); + if (ret) + return ret; + + /* If there are no other options, try atomic_test_and_set if the value + being stored is 1. */ + if (val == const1_rtx) + ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE); + + return ret; +} + +/* This function expands the atomic test_and_set operation: + atomically store a boolean TRUE into MEM and return the previous value. + + MEMMODEL is the memory model variant to use. + TARGET is an optional place to stick the return value. */ + +rtx +expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model) +{ + machine_mode mode = GET_MODE (mem); + rtx ret, trueval, subtarget; + + ret = maybe_emit_atomic_test_and_set (target, mem, model); + if (ret) + return ret; + + /* Be binary compatible with non-default settings of trueval, and different + cpu revisions. E.g. one revision may have atomic-test-and-set, but + another only has atomic-exchange. */ + if (targetm.atomic_test_and_set_trueval == 1) + { + trueval = const1_rtx; + subtarget = target ? target : gen_reg_rtx (mode); + } + else + { + trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode); + subtarget = gen_reg_rtx (mode); + } + + /* Try the atomic-exchange optab... */ + ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model); + + /* ... then an atomic-compare-and-swap loop ... */ + if (!ret) + ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval); + + /* ... before trying the vaguely defined legacy lock_test_and_set. */ + if (!ret) + ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model); + + /* Recall that the legacy lock_test_and_set optab was allowed to do magic + things with the value 1. Thus we try again without trueval. */ + if (!ret && targetm.atomic_test_and_set_trueval != 1) + ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model); + + /* Failing all else, assume a single threaded environment and simply + perform the operation. */ + if (!ret) + { + /* If the result is ignored skip the move to target. */ + if (subtarget != const0_rtx) + emit_move_insn (subtarget, mem); + + emit_move_insn (mem, trueval); + ret = subtarget; + } + + /* Recall that have to return a boolean value; rectify if trueval + is not exactly one. */ + if (targetm.atomic_test_and_set_trueval != 1) + ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1); + + return ret; +} + +/* This function expands the atomic exchange operation: + atomically store VAL in MEM and return the previous value in MEM. + + MEMMODEL is the memory model variant to use. + TARGET is an optional place to stick the return value. */ + +rtx +expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model) +{ + machine_mode mode = GET_MODE (mem); + rtx ret; + + /* If loads are not atomic for the required size and we are not called to + provide a __sync builtin, do not do anything so that we stay consistent + with atomic loads of the same size. */ + if (!can_atomic_load_p (mode) && !is_mm_sync (model)) + return NULL_RTX; + + ret = maybe_emit_atomic_exchange (target, mem, val, model); + + /* Next try a compare-and-swap loop for the exchange. */ + if (!ret) + ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val); + + return ret; +} + +/* This function expands the atomic compare exchange operation: + + *PTARGET_BOOL is an optional place to store the boolean success/failure. + *PTARGET_OVAL is an optional place to store the old value from memory. + Both target parameters may be NULL or const0_rtx to indicate that we do + not care about that return value. Both target parameters are updated on + success to the actual location of the corresponding result. + + MEMMODEL is the memory model variant to use. + + The return value of the function is true for success. */ + +bool +expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval, + rtx mem, rtx expected, rtx desired, + bool is_weak, enum memmodel succ_model, + enum memmodel fail_model) +{ + machine_mode mode = GET_MODE (mem); + struct expand_operand ops[8]; + enum insn_code icode; + rtx target_oval, target_bool = NULL_RTX; + rtx libfunc; + + /* If loads are not atomic for the required size and we are not called to + provide a __sync builtin, do not do anything so that we stay consistent + with atomic loads of the same size. */ + if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model)) + return false; + + /* Load expected into a register for the compare and swap. */ + if (MEM_P (expected)) + expected = copy_to_reg (expected); + + /* Make sure we always have some place to put the return oldval. + Further, make sure that place is distinct from the input expected, + just in case we need that path down below. */ + if (ptarget_oval && *ptarget_oval == const0_rtx) + ptarget_oval = NULL; + + if (ptarget_oval == NULL + || (target_oval = *ptarget_oval) == NULL + || reg_overlap_mentioned_p (expected, target_oval)) + target_oval = gen_reg_rtx (mode); + + icode = direct_optab_handler (atomic_compare_and_swap_optab, mode); + if (icode != CODE_FOR_nothing) + { + machine_mode bool_mode = insn_data[icode].operand[0].mode; + + if (ptarget_bool && *ptarget_bool == const0_rtx) + ptarget_bool = NULL; + + /* Make sure we always have a place for the bool operand. */ + if (ptarget_bool == NULL + || (target_bool = *ptarget_bool) == NULL + || GET_MODE (target_bool) != bool_mode) + target_bool = gen_reg_rtx (bool_mode); + + /* Emit the compare_and_swap. */ + create_output_operand (&ops[0], target_bool, bool_mode); + create_output_operand (&ops[1], target_oval, mode); + create_fixed_operand (&ops[2], mem); + create_input_operand (&ops[3], expected, mode); + create_input_operand (&ops[4], desired, mode); + create_integer_operand (&ops[5], is_weak); + create_integer_operand (&ops[6], succ_model); + create_integer_operand (&ops[7], fail_model); + if (maybe_expand_insn (icode, 8, ops)) + { + /* Return success/failure. */ + target_bool = ops[0].value; + target_oval = ops[1].value; + goto success; + } + } + + /* Otherwise fall back to the original __sync_val_compare_and_swap + which is always seq-cst. */ + icode = optab_handler (sync_compare_and_swap_optab, mode); + if (icode != CODE_FOR_nothing) + { + rtx cc_reg; + + create_output_operand (&ops[0], target_oval, mode); + create_fixed_operand (&ops[1], mem); + create_input_operand (&ops[2], expected, mode); + create_input_operand (&ops[3], desired, mode); + if (!maybe_expand_insn (icode, 4, ops)) + return false; + + target_oval = ops[0].value; + + /* If the caller isn't interested in the boolean return value, + skip the computation of it. */ + if (ptarget_bool == NULL) + goto success; + + /* Otherwise, work out if the compare-and-swap succeeded. */ + cc_reg = NULL_RTX; + if (have_insn_for (COMPARE, CCmode)) + note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); + if (cc_reg) + { + target_bool = emit_store_flag_force (target_bool, EQ, cc_reg, + const0_rtx, VOIDmode, 0, 1); + goto success; + } + goto success_bool_from_val; + } + + /* Also check for library support for __sync_val_compare_and_swap. */ + libfunc = optab_libfunc (sync_compare_and_swap_optab, mode); + if (libfunc != NULL) + { + rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); + rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL, + mode, addr, ptr_mode, + expected, mode, desired, mode); + emit_move_insn (target_oval, target); + + /* Compute the boolean return value only if requested. */ + if (ptarget_bool) + goto success_bool_from_val; + else + goto success; + } + + /* Failure. */ + return false; + + success_bool_from_val: + target_bool = emit_store_flag_force (target_bool, EQ, target_oval, + expected, VOIDmode, 1, 1); + success: + /* Make sure that the oval output winds up where the caller asked. */ + if (ptarget_oval) + *ptarget_oval = target_oval; + if (ptarget_bool) + *ptarget_bool = target_bool; + return true; +} + +/* Generate asm volatile("" : : : "memory") as the memory blockage. */ + +static void +expand_asm_memory_blockage (void) +{ + rtx asm_op, clob; + + asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0, + rtvec_alloc (0), rtvec_alloc (0), + rtvec_alloc (0), UNKNOWN_LOCATION); + MEM_VOLATILE_P (asm_op) = 1; + + clob = gen_rtx_SCRATCH (VOIDmode); + clob = gen_rtx_MEM (BLKmode, clob); + clob = gen_rtx_CLOBBER (VOIDmode, clob); + + emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob))); +} + +/* Do not propagate memory accesses across this point. */ + +static void +expand_memory_blockage (void) +{ + if (targetm.have_memory_blockage ()) + emit_insn (targetm.gen_memory_blockage ()); + else + expand_asm_memory_blockage (); +} + +/* This routine will either emit the mem_thread_fence pattern or issue a + sync_synchronize to generate a fence for memory model MEMMODEL. */ + +void +expand_mem_thread_fence (enum memmodel model) +{ + if (is_mm_relaxed (model)) + return; + if (targetm.have_mem_thread_fence ()) + { + emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model))); + expand_memory_blockage (); + } + else if (targetm.have_memory_barrier ()) + emit_insn (targetm.gen_memory_barrier ()); + else if (synchronize_libfunc != NULL_RTX) + emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode); + else + expand_memory_blockage (); +} + +/* Emit a signal fence with given memory model. */ + +void +expand_mem_signal_fence (enum memmodel model) +{ + /* No machine barrier is required to implement a signal fence, but + a compiler memory barrier must be issued, except for relaxed MM. */ + if (!is_mm_relaxed (model)) + expand_memory_blockage (); +} + +/* This function expands the atomic load operation: + return the atomically loaded value in MEM. + + MEMMODEL is the memory model variant to use. + TARGET is an option place to stick the return value. */ + +rtx +expand_atomic_load (rtx target, rtx mem, enum memmodel model) +{ + machine_mode mode = GET_MODE (mem); + enum insn_code icode; + + /* If the target supports the load directly, great. */ + icode = direct_optab_handler (atomic_load_optab, mode); + if (icode != CODE_FOR_nothing) + { + struct expand_operand ops[3]; + rtx_insn *last = get_last_insn (); + if (is_mm_seq_cst (model)) + expand_memory_blockage (); + + create_output_operand (&ops[0], target, mode); + create_fixed_operand (&ops[1], mem); + create_integer_operand (&ops[2], model); + if (maybe_expand_insn (icode, 3, ops)) + { + if (!is_mm_relaxed (model)) + expand_memory_blockage (); + return ops[0].value; + } + delete_insns_since (last); + } + + /* If the size of the object is greater than word size on this target, + then we assume that a load will not be atomic. We could try to + emulate a load with a compare-and-swap operation, but the store that + doing this could result in would be incorrect if this is a volatile + atomic load or targetting read-only-mapped memory. */ + if (GET_MODE_PRECISION (mode) > BITS_PER_WORD) + /* If there is no atomic load, leave the library call. */ + return NULL_RTX; + + /* Otherwise assume loads are atomic, and emit the proper barriers. */ + if (!target || target == const0_rtx) + target = gen_reg_rtx (mode); + + /* For SEQ_CST, emit a barrier before the load. */ + if (is_mm_seq_cst (model)) + expand_mem_thread_fence (model); + + emit_move_insn (target, mem); + + /* Emit the appropriate barrier after the load. */ + expand_mem_thread_fence (model); + + return target; +} + +/* This function expands the atomic store operation: + Atomically store VAL in MEM. + MEMMODEL is the memory model variant to use. + USE_RELEASE is true if __sync_lock_release can be used as a fall back. + function returns const0_rtx if a pattern was emitted. */ + +rtx +expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release) +{ + machine_mode mode = GET_MODE (mem); + enum insn_code icode; + struct expand_operand ops[3]; + + /* If the target supports the store directly, great. */ + icode = direct_optab_handler (atomic_store_optab, mode); + if (icode != CODE_FOR_nothing) + { + rtx_insn *last = get_last_insn (); + if (!is_mm_relaxed (model)) + expand_memory_blockage (); + create_fixed_operand (&ops[0], mem); + create_input_operand (&ops[1], val, mode); + create_integer_operand (&ops[2], model); + if (maybe_expand_insn (icode, 3, ops)) + { + if (is_mm_seq_cst (model)) + expand_memory_blockage (); + return const0_rtx; + } + delete_insns_since (last); + } + + /* If using __sync_lock_release is a viable alternative, try it. + Note that this will not be set to true if we are expanding a generic + __atomic_store_n. */ + if (use_release) + { + icode = direct_optab_handler (sync_lock_release_optab, mode); + if (icode != CODE_FOR_nothing) + { + create_fixed_operand (&ops[0], mem); + create_input_operand (&ops[1], const0_rtx, mode); + if (maybe_expand_insn (icode, 2, ops)) + { + /* lock_release is only a release barrier. */ + if (is_mm_seq_cst (model)) + expand_mem_thread_fence (model); + return const0_rtx; + } + } + } + + /* If the size of the object is greater than word size on this target, + a default store will not be atomic. */ + if (GET_MODE_PRECISION (mode) > BITS_PER_WORD) + { + /* If loads are atomic or we are called to provide a __sync builtin, + we can try a atomic_exchange and throw away the result. Otherwise, + don't do anything so that we do not create an inconsistency between + loads and stores. */ + if (can_atomic_load_p (mode) || is_mm_sync (model)) + { + rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model); + if (!target) + target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, + val); + if (target) + return const0_rtx; + } + return NULL_RTX; + } + + /* Otherwise assume stores are atomic, and emit the proper barriers. */ + expand_mem_thread_fence (model); + + emit_move_insn (mem, val); + + /* For SEQ_CST, also emit a barrier after the store. */ + if (is_mm_seq_cst (model)) + expand_mem_thread_fence (model); + + return const0_rtx; +} + + +/* Structure containing the pointers and values required to process the + various forms of the atomic_fetch_op and atomic_op_fetch builtins. */ + +struct atomic_op_functions +{ + direct_optab mem_fetch_before; + direct_optab mem_fetch_after; + direct_optab mem_no_result; + optab fetch_before; + optab fetch_after; + direct_optab no_result; + enum rtx_code reverse_code; +}; + + +/* Fill in structure pointed to by OP with the various optab entries for an + operation of type CODE. */ + +static void +get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code) +{ + gcc_assert (op!= NULL); + + /* If SWITCHABLE_TARGET is defined, then subtargets can be switched + in the source code during compilation, and the optab entries are not + computable until runtime. Fill in the values at runtime. */ + switch (code) + { + case PLUS: + op->mem_fetch_before = atomic_fetch_add_optab; + op->mem_fetch_after = atomic_add_fetch_optab; + op->mem_no_result = atomic_add_optab; + op->fetch_before = sync_old_add_optab; + op->fetch_after = sync_new_add_optab; + op->no_result = sync_add_optab; + op->reverse_code = MINUS; + break; + case MINUS: + op->mem_fetch_before = atomic_fetch_sub_optab; + op->mem_fetch_after = atomic_sub_fetch_optab; + op->mem_no_result = atomic_sub_optab; + op->fetch_before = sync_old_sub_optab; + op->fetch_after = sync_new_sub_optab; + op->no_result = sync_sub_optab; + op->reverse_code = PLUS; + break; + case XOR: + op->mem_fetch_before = atomic_fetch_xor_optab; + op->mem_fetch_after = atomic_xor_fetch_optab; + op->mem_no_result = atomic_xor_optab; + op->fetch_before = sync_old_xor_optab; + op->fetch_after = sync_new_xor_optab; + op->no_result = sync_xor_optab; + op->reverse_code = XOR; + break; + case AND: + op->mem_fetch_before = atomic_fetch_and_optab; + op->mem_fetch_after = atomic_and_fetch_optab; + op->mem_no_result = atomic_and_optab; + op->fetch_before = sync_old_and_optab; + op->fetch_after = sync_new_and_optab; + op->no_result = sync_and_optab; + op->reverse_code = UNKNOWN; + break; + case IOR: + op->mem_fetch_before = atomic_fetch_or_optab; + op->mem_fetch_after = atomic_or_fetch_optab; + op->mem_no_result = atomic_or_optab; + op->fetch_before = sync_old_ior_optab; + op->fetch_after = sync_new_ior_optab; + op->no_result = sync_ior_optab; + op->reverse_code = UNKNOWN; + break; + case NOT: + op->mem_fetch_before = atomic_fetch_nand_optab; + op->mem_fetch_after = atomic_nand_fetch_optab; + op->mem_no_result = atomic_nand_optab; + op->fetch_before = sync_old_nand_optab; + op->fetch_after = sync_new_nand_optab; + op->no_result = sync_nand_optab; + op->reverse_code = UNKNOWN; + break; + default: + gcc_unreachable (); + } +} + +/* See if there is a more optimal way to implement the operation "*MEM CODE VAL" + using memory order MODEL. If AFTER is true the operation needs to return + the value of *MEM after the operation, otherwise the previous value. + TARGET is an optional place to place the result. The result is unused if + it is const0_rtx. + Return the result if there is a better sequence, otherwise NULL_RTX. */ + +static rtx +maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code, + enum memmodel model, bool after) +{ + /* If the value is prefetched, or not used, it may be possible to replace + the sequence with a native exchange operation. */ + if (!after || target == const0_rtx) + { + /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */ + if (code == AND && val == const0_rtx) + { + if (target == const0_rtx) + target = gen_reg_rtx (GET_MODE (mem)); + return maybe_emit_atomic_exchange (target, mem, val, model); + } + + /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */ + if (code == IOR && val == constm1_rtx) + { + if (target == const0_rtx) + target = gen_reg_rtx (GET_MODE (mem)); + return maybe_emit_atomic_exchange (target, mem, val, model); + } + } + + return NULL_RTX; +} + +/* Try to emit an instruction for a specific operation varaition. + OPTAB contains the OP functions. + TARGET is an optional place to return the result. const0_rtx means unused. + MEM is the memory location to operate on. + VAL is the value to use in the operation. + USE_MEMMODEL is TRUE if the variation with a memory model should be tried. + MODEL is the memory model, if used. + AFTER is true if the returned result is the value after the operation. */ + +static rtx +maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem, + rtx val, bool use_memmodel, enum memmodel model, bool after) +{ + machine_mode mode = GET_MODE (mem); + struct expand_operand ops[4]; + enum insn_code icode; + int op_counter = 0; + int num_ops; + + /* Check to see if there is a result returned. */ + if (target == const0_rtx) + { + if (use_memmodel) + { + icode = direct_optab_handler (optab->mem_no_result, mode); + create_integer_operand (&ops[2], model); + num_ops = 3; + } + else + { + icode = direct_optab_handler (optab->no_result, mode); + num_ops = 2; + } + } + /* Otherwise, we need to generate a result. */ + else + { + if (use_memmodel) + { + icode = direct_optab_handler (after ? optab->mem_fetch_after + : optab->mem_fetch_before, mode); + create_integer_operand (&ops[3], model); + num_ops = 4; + } + else + { + icode = optab_handler (after ? optab->fetch_after + : optab->fetch_before, mode); + num_ops = 3; + } + create_output_operand (&ops[op_counter++], target, mode); + } + if (icode == CODE_FOR_nothing) + return NULL_RTX; + + create_fixed_operand (&ops[op_counter++], mem); + /* VAL may have been promoted to a wider mode. Shrink it if so. */ + create_convert_operand_to (&ops[op_counter++], val, mode, true); + + if (maybe_expand_insn (icode, num_ops, ops)) + return (target == const0_rtx ? const0_rtx : ops[0].value); + + return NULL_RTX; +} + + +/* This function expands an atomic fetch_OP or OP_fetch operation: + TARGET is an option place to stick the return value. const0_rtx indicates + the result is unused. + atomically fetch MEM, perform the operation with VAL and return it to MEM. + CODE is the operation being performed (OP) + MEMMODEL is the memory model variant to use. + AFTER is true to return the result of the operation (OP_fetch). + AFTER is false to return the value before the operation (fetch_OP). + + This function will *only* generate instructions if there is a direct + optab. No compare and swap loops or libcalls will be generated. */ + +static rtx +expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val, + enum rtx_code code, enum memmodel model, + bool after) +{ + machine_mode mode = GET_MODE (mem); + struct atomic_op_functions optab; + rtx result; + bool unused_result = (target == const0_rtx); + + get_atomic_op_for_code (&optab, code); + + /* Check to see if there are any better instructions. */ + result = maybe_optimize_fetch_op (target, mem, val, code, model, after); + if (result) + return result; + + /* Check for the case where the result isn't used and try those patterns. */ + if (unused_result) + { + /* Try the memory model variant first. */ + result = maybe_emit_op (&optab, target, mem, val, true, model, true); + if (result) + return result; + + /* Next try the old style withuot a memory model. */ + result = maybe_emit_op (&optab, target, mem, val, false, model, true); + if (result) + return result; + + /* There is no no-result pattern, so try patterns with a result. */ + target = NULL_RTX; + } + + /* Try the __atomic version. */ + result = maybe_emit_op (&optab, target, mem, val, true, model, after); + if (result) + return result; + + /* Try the older __sync version. */ + result = maybe_emit_op (&optab, target, mem, val, false, model, after); + if (result) + return result; + + /* If the fetch value can be calculated from the other variation of fetch, + try that operation. */ + if (after || unused_result || optab.reverse_code != UNKNOWN) + { + /* Try the __atomic version, then the older __sync version. */ + result = maybe_emit_op (&optab, target, mem, val, true, model, !after); + if (!result) + result = maybe_emit_op (&optab, target, mem, val, false, model, !after); + + if (result) + { + /* If the result isn't used, no need to do compensation code. */ + if (unused_result) + return result; + + /* Issue compensation code. Fetch_after == fetch_before OP val. + Fetch_before == after REVERSE_OP val. */ + if (!after) + code = optab.reverse_code; + if (code == NOT) + { + result = expand_simple_binop (mode, AND, result, val, NULL_RTX, + true, OPTAB_LIB_WIDEN); + result = expand_simple_unop (mode, NOT, result, target, true); + } + else + result = expand_simple_binop (mode, code, result, val, target, + true, OPTAB_LIB_WIDEN); + return result; + } + } + + /* No direct opcode can be generated. */ + return NULL_RTX; +} + + + +/* This function expands an atomic fetch_OP or OP_fetch operation: + TARGET is an option place to stick the return value. const0_rtx indicates + the result is unused. + atomically fetch MEM, perform the operation with VAL and return it to MEM. + CODE is the operation being performed (OP) + MEMMODEL is the memory model variant to use. + AFTER is true to return the result of the operation (OP_fetch). + AFTER is false to return the value before the operation (fetch_OP). */ +rtx +expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code, + enum memmodel model, bool after) +{ + machine_mode mode = GET_MODE (mem); + rtx result; + bool unused_result = (target == const0_rtx); + + /* If loads are not atomic for the required size and we are not called to + provide a __sync builtin, do not do anything so that we stay consistent + with atomic loads of the same size. */ + if (!can_atomic_load_p (mode) && !is_mm_sync (model)) + return NULL_RTX; + + result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model, + after); + + if (result) + return result; + + /* Add/sub can be implemented by doing the reverse operation with -(val). */ + if (code == PLUS || code == MINUS) + { + rtx tmp; + enum rtx_code reverse = (code == PLUS ? MINUS : PLUS); + + start_sequence (); + tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true); + result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse, + model, after); + if (result) + { + /* PLUS worked so emit the insns and return. */ + tmp = get_insns (); + end_sequence (); + emit_insn (tmp); + return result; + } + + /* PLUS did not work, so throw away the negation code and continue. */ + end_sequence (); + } + + /* Try the __sync libcalls only if we can't do compare-and-swap inline. */ + if (!can_compare_and_swap_p (mode, false)) + { + rtx libfunc; + bool fixup = false; + enum rtx_code orig_code = code; + struct atomic_op_functions optab; + + get_atomic_op_for_code (&optab, code); + libfunc = optab_libfunc (after ? optab.fetch_after + : optab.fetch_before, mode); + if (libfunc == NULL + && (after || unused_result || optab.reverse_code != UNKNOWN)) + { + fixup = true; + if (!after) + code = optab.reverse_code; + libfunc = optab_libfunc (after ? optab.fetch_before + : optab.fetch_after, mode); + } + if (libfunc != NULL) + { + rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); + result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode, + addr, ptr_mode, val, mode); + + if (!unused_result && fixup) + result = expand_simple_binop (mode, code, result, val, target, + true, OPTAB_LIB_WIDEN); + return result; + } + + /* We need the original code for any further attempts. */ + code = orig_code; + } + + /* If nothing else has succeeded, default to a compare and swap loop. */ + if (can_compare_and_swap_p (mode, true)) + { + rtx_insn *insn; + rtx t0 = gen_reg_rtx (mode), t1; + + start_sequence (); + + /* If the result is used, get a register for it. */ + if (!unused_result) + { + if (!target || !register_operand (target, mode)) + target = gen_reg_rtx (mode); + /* If fetch_before, copy the value now. */ + if (!after) + emit_move_insn (target, t0); + } + else + target = const0_rtx; + + t1 = t0; + if (code == NOT) + { + t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX, + true, OPTAB_LIB_WIDEN); + t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true); + } + else + t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true, + OPTAB_LIB_WIDEN); + + /* For after, copy the value now. */ + if (!unused_result && after) + emit_move_insn (target, t1); + insn = get_insns (); + end_sequence (); + + if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn)) + return target; + } + + return NULL_RTX; +} + +/* Return true if OPERAND is suitable for operand number OPNO of + instruction ICODE. */ + +bool +insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand) +{ + return (!insn_data[(int) icode].operand[opno].predicate + || (insn_data[(int) icode].operand[opno].predicate + (operand, insn_data[(int) icode].operand[opno].mode))); +} + +/* TARGET is a target of a multiword operation that we are going to + implement as a series of word-mode operations. Return true if + TARGET is suitable for this purpose. */ + +bool +valid_multiword_target_p (rtx target) +{ + machine_mode mode; + int i; + + mode = GET_MODE (target); + for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD) + if (!validate_subreg (word_mode, mode, target, i)) + return false; + return true; +} + +/* Like maybe_legitimize_operand, but do not change the code of the + current rtx value. */ + +static bool +maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno, + struct expand_operand *op) +{ + /* See if the operand matches in its current form. */ + if (insn_operand_matches (icode, opno, op->value)) + return true; + + /* If the operand is a memory whose address has no side effects, + try forcing the address into a non-virtual pseudo register. + The check for side effects is important because copy_to_mode_reg + cannot handle things like auto-modified addresses. */ + if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value)) + { + rtx addr, mem; + + mem = op->value; + addr = XEXP (mem, 0); + if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER) + && !side_effects_p (addr)) + { + rtx_insn *last; + machine_mode mode; + + last = get_last_insn (); + mode = get_address_mode (mem); + mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr)); + if (insn_operand_matches (icode, opno, mem)) + { + op->value = mem; + return true; + } + delete_insns_since (last); + } + } + + return false; +} + +/* Try to make OP match operand OPNO of instruction ICODE. Return true + on success, storing the new operand value back in OP. */ + +static bool +maybe_legitimize_operand (enum insn_code icode, unsigned int opno, + struct expand_operand *op) +{ + machine_mode mode, imode; + bool old_volatile_ok, result; + + mode = op->mode; + switch (op->type) + { + case EXPAND_FIXED: + old_volatile_ok = volatile_ok; + volatile_ok = true; + result = maybe_legitimize_operand_same_code (icode, opno, op); + volatile_ok = old_volatile_ok; + return result; + + case EXPAND_OUTPUT: + gcc_assert (mode != VOIDmode); + if (op->value + && op->value != const0_rtx + && GET_MODE (op->value) == mode + && maybe_legitimize_operand_same_code (icode, opno, op)) + return true; + + op->value = gen_reg_rtx (mode); + op->target = 0; + break; + + case EXPAND_INPUT: + input: + gcc_assert (mode != VOIDmode); + gcc_assert (GET_MODE (op->value) == VOIDmode + || GET_MODE (op->value) == mode); + if (maybe_legitimize_operand_same_code (icode, opno, op)) + return true; + + op->value = copy_to_mode_reg (mode, op->value); + break; + + case EXPAND_CONVERT_TO: + gcc_assert (mode != VOIDmode); + op->value = convert_to_mode (mode, op->value, op->unsigned_p); + goto input; + + case EXPAND_CONVERT_FROM: + if (GET_MODE (op->value) != VOIDmode) + mode = GET_MODE (op->value); + else + /* The caller must tell us what mode this value has. */ + gcc_assert (mode != VOIDmode); + + imode = insn_data[(int) icode].operand[opno].mode; + if (imode != VOIDmode && imode != mode) + { + op->value = convert_modes (imode, mode, op->value, op->unsigned_p); + mode = imode; + } + goto input; + + case EXPAND_ADDRESS: + op->value = convert_memory_address (as_a <scalar_int_mode> (mode), + op->value); + goto input; + + case EXPAND_INTEGER: + mode = insn_data[(int) icode].operand[opno].mode; + if (mode != VOIDmode && const_int_operand (op->value, mode)) + goto input; + break; + } + return insn_operand_matches (icode, opno, op->value); +} + +/* Make OP describe an input operand that should have the same value + as VALUE, after any mode conversion that the target might request. + TYPE is the type of VALUE. */ + +void +create_convert_operand_from_type (struct expand_operand *op, + rtx value, tree type) +{ + create_convert_operand_from (op, value, TYPE_MODE (type), + TYPE_UNSIGNED (type)); +} + +/* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS) + of instruction ICODE. Return true on success, leaving the new operand + values in the OPS themselves. Emit no code on failure. */ + +bool +maybe_legitimize_operands (enum insn_code icode, unsigned int opno, + unsigned int nops, struct expand_operand *ops) +{ + rtx_insn *last; + unsigned int i; + + last = get_last_insn (); + for (i = 0; i < nops; i++) + if (!maybe_legitimize_operand (icode, opno + i, &ops[i])) + { + delete_insns_since (last); + return false; + } + return true; +} + +/* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS) + as its operands. Return the instruction pattern on success, + and emit any necessary set-up code. Return null and emit no + code on failure. */ + +rtx_insn * +maybe_gen_insn (enum insn_code icode, unsigned int nops, + struct expand_operand *ops) +{ + gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args); + if (!maybe_legitimize_operands (icode, 0, nops, ops)) + return NULL; + + switch (nops) + { + case 1: + return GEN_FCN (icode) (ops[0].value); + case 2: + return GEN_FCN (icode) (ops[0].value, ops[1].value); + case 3: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value); + case 4: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value); + case 5: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value, ops[4].value); + case 6: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value, ops[4].value, ops[5].value); + case 7: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value, ops[4].value, ops[5].value, + ops[6].value); + case 8: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value, ops[4].value, ops[5].value, + ops[6].value, ops[7].value); + case 9: + return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, + ops[3].value, ops[4].value, ops[5].value, + ops[6].value, ops[7].value, ops[8].value); + } + gcc_unreachable (); +} + +/* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS) + as its operands. Return true on success and emit no code on failure. */ + +bool +maybe_expand_insn (enum insn_code icode, unsigned int nops, + struct expand_operand *ops) +{ + rtx_insn *pat = maybe_gen_insn (icode, nops, ops); + if (pat) + { + emit_insn (pat); + return true; + } + return false; +} + +/* Like maybe_expand_insn, but for jumps. */ + +bool +maybe_expand_jump_insn (enum insn_code icode, unsigned int nops, + struct expand_operand *ops) +{ + rtx_insn *pat = maybe_gen_insn (icode, nops, ops); + if (pat) + { + emit_jump_insn (pat); + return true; + } + return false; +} + +/* Emit instruction ICODE, using operands [OPS, OPS + NOPS) + as its operands. */ + +void +expand_insn (enum insn_code icode, unsigned int nops, + struct expand_operand *ops) +{ + if (!maybe_expand_insn (icode, nops, ops)) + gcc_unreachable (); +} + +/* Like expand_insn, but for jumps. */ + +void +expand_jump_insn (enum insn_code icode, unsigned int nops, + struct expand_operand *ops) +{ + if (!maybe_expand_jump_insn (icode, nops, ops)) + gcc_unreachable (); +}